src/ostree/ot-builtin-ls.c \
src/ostree/ot-builtin-prune.c \
src/ostree/ot-builtin-remote.c \
- src/ostree/ot-builtin-pack.c \
- src/ostree/ot-builtin-unpack.c \
src/ostree/ot-builtin-rev-parse.c \
src/ostree/ot-builtin-show.c \
src/ostree/ot-builtin-write-refs.c \
return g_string_free (path, FALSE);
}
-static char *
-get_pack_name (gboolean is_meta,
- gboolean is_index,
- const char *prefix,
- const char *checksum)
-{
- GString *path;
-
- g_assert (strlen (checksum) == 64);
-
- path = g_string_new (prefix);
- if (is_meta)
- g_string_append (path, "ostmetapack-");
- else
- g_string_append (path, "ostdatapack-");
- g_string_append (path, checksum);
- if (is_index)
- g_string_append (path, ".index");
- else
- g_string_append (path, ".data");
-
- return g_string_free (path, FALSE);
-}
-
-char *
-ostree_get_pack_index_name (gboolean is_meta,
- const char *checksum)
-{
- return get_pack_name (is_meta, TRUE, "", checksum);
-}
-
-char *
-ostree_get_pack_data_name (gboolean is_meta,
- const char *checksum)
-{
- return get_pack_name (is_meta, FALSE, "", checksum);
-}
-
-char *
-ostree_get_relative_pack_index_path (gboolean is_meta,
- const char *checksum)
-{
- return get_pack_name (is_meta, TRUE, "objects/pack/", checksum);
-}
-
-char *
-ostree_get_relative_pack_data_path (gboolean is_meta,
- const char *checksum)
-{
- return get_pack_name (is_meta, FALSE, "objects/pack/", checksum);
-}
-
gboolean
ostree_file_header_parse (GVariant *metadata,
GFileInfo **out_file_info,
return ret;
}
-gboolean
-ostree_read_pack_entry_raw (guchar *pack_data,
- guint64 pack_len,
- guint64 offset,
- gboolean trusted,
- gboolean is_meta,
- GVariant **out_entry,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint64 entry_start;
- guint64 entry_end;
- guint32 entry_len;
- ot_lvariant GVariant *ret_entry = NULL;
-
- if (G_UNLIKELY (!(offset <= pack_len)))
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Corrupted pack index; out of range offset %" G_GUINT64_FORMAT,
- offset);
- goto out;
- }
- if (G_UNLIKELY (!((offset & 0x3) == 0)))
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT,
- offset);
- goto out;
- }
-
- entry_start = ALIGN_VALUE (offset + 4, 8);
- if (G_UNLIKELY (!(entry_start <= pack_len)))
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
- entry_start);
- goto out;
- }
-
- g_assert ((((guint64)pack_data+offset) & 0x3) == 0);
- entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
-
- entry_end = entry_start + entry_len;
- if (G_UNLIKELY (!(entry_end <= pack_len)))
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Corrupted pack index; out of range entry length %u",
- entry_len);
- goto out;
- }
-
- ret_entry = g_variant_new_from_data (is_meta ? OSTREE_PACK_META_FILE_VARIANT_FORMAT :
- OSTREE_PACK_DATA_FILE_VARIANT_FORMAT,
- pack_data+entry_start, entry_len,
- trusted, NULL, NULL);
- g_variant_ref_sink (ret_entry);
- ret = TRUE;
- ot_transfer_out_value (out_entry, &ret_entry);
- out:
- return ret;
-}
-
-gboolean
-ostree_parse_file_pack_entry (GVariant *pack_entry,
- GInputStream **out_input,
- GFileInfo **out_info,
- GVariant **out_xattrs,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guchar entry_flags;
- ot_lvariant GVariant *file_header = NULL;
- ot_lvariant GVariant *pack_data = NULL;
- ot_lobj GInputStream *memory_input = NULL;
- ot_lobj GInputStream *ret_input = NULL;
- ot_lobj GFileInfo *ret_info = NULL;
- ot_lvariant GVariant *ret_xattrs = NULL;
-
- g_variant_get_child (pack_entry, 1, "y", &entry_flags);
- g_variant_get_child (pack_entry, 2, "@(uuuusa(ayay))", &file_header);
- g_variant_get_child (pack_entry, 3, "@ay", &pack_data);
-
- if (!ostree_file_header_parse (file_header, &ret_info, &ret_xattrs,
- error))
- goto out;
- g_file_info_set_size (ret_info, g_variant_get_size (pack_data));
-
- if (g_file_info_get_file_type (ret_info) == G_FILE_TYPE_REGULAR)
- {
- memory_input = ot_variant_read (pack_data);
-
- if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
- {
- ot_lobj GConverter *decompressor = NULL;
-
- decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP);
- ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
- "converter", decompressor,
- "base-stream", memory_input,
- "close-base-stream", TRUE,
- NULL);
- }
- else
- {
- ret_input = g_object_ref (memory_input);
- }
- }
-
- ret = TRUE;
- ot_transfer_out_value (out_input, &ret_input);
- ot_transfer_out_value (out_info, &ret_info);
- ot_transfer_out_value (out_xattrs, &ret_xattrs);
- out:
- return ret;
-}
-
-gboolean
-ostree_pack_index_search (GVariant *index,
- GVariant *csum_v,
- OstreeObjectType objtype,
- guint64 *out_offset)
-{
- gboolean ret = FALSE;
- gsize imax, imin;
- gsize n;
- guint32 target_objtype;
- const guchar *csum;
- ot_lvariant GVariant *index_contents = NULL;
-
- csum = ostree_checksum_bytes_peek (csum_v);
-
- index_contents = g_variant_get_child_value (index, 2);
-
- target_objtype = (guint32) objtype;
-
- n = g_variant_n_children (index_contents);
-
- if (n == 0)
- goto out;
-
- imax = n - 1;
- imin = 0;
- while (imax >= imin)
- {
- GVariant *cur_csum_bytes;
- guchar cur_objtype;
- guint64 cur_offset;
- gsize imid;
- int c;
-
- imid = (imin + imax) / 2;
-
- g_variant_get_child (index_contents, imid, "(y@ayt)", &cur_objtype,
- &cur_csum_bytes, &cur_offset);
-
- c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum);
- if (c == 0)
- {
- if (cur_objtype < target_objtype)
- c = -1;
- else if (cur_objtype > target_objtype)
- c = 1;
- }
- g_variant_unref (cur_csum_bytes);
-
- if (c < 0)
- imin = imid + 1;
- else if (c > 0)
- {
- if (imid == 0)
- goto out;
- imax = imid - 1;
- }
- else
- {
- if (out_offset)
- *out_offset = GUINT64_FROM_BE (cur_offset);
- ret = TRUE;
- goto out;
- }
- }
-
- out:
- return ret;
-}
-
gboolean
ostree_validate_structureof_objtype (guchar objtype,
GError **error)
out:
return ret;
}
-
-gboolean
-ostree_validate_structureof_pack_index (GVariant *index,
- GError **error)
-{
- gboolean ret = FALSE;
- const char *header;
- guchar objtype_u8;
- guint64 offset;
- ot_lvariant GVariant *csum_v = NULL;
- GVariantIter *content_iter = NULL;
-
- if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error))
- goto out;
-
- g_variant_get_child (index, 0, "&s", &header);
-
- if (strcmp (header, "OSTv0PACKINDEX") != 0)
- {
- g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid pack index; doesn't match header");
- goto out;
- }
-
- g_variant_get_child (index, 2, "a(yayt)", &content_iter);
-
- while (g_variant_iter_loop (content_iter, "(y@ayt)",
- &objtype_u8, &csum_v, &offset))
- {
- if (!ostree_validate_structureof_objtype (objtype_u8, error))
- goto out;
- if (!ostree_validate_structureof_csum_v (csum_v, error))
- goto out;
- }
- csum_v = NULL;
-
- ret = TRUE;
- out:
- if (content_iter)
- g_variant_iter_free (content_iter);
- return ret;
-}
-
-gboolean
-ostree_validate_structureof_pack_superindex (GVariant *superindex,
- GError **error)
-{
- gboolean ret = FALSE;
- const char *header;
- ot_lvariant GVariant *csum_v = NULL;
- ot_lvariant GVariant *bloom = NULL;
- GVariantIter *content_iter = NULL;
-
- if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error))
- goto out;
-
- g_variant_get_child (superindex, 0, "&s", &header);
-
- if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0)
- {
- g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid pack superindex; doesn't match header");
- goto out;
- }
-
- g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
- while (g_variant_iter_loop (content_iter, "(@ay@ay)",
- &csum_v, &bloom))
- {
- if (!ostree_validate_structureof_csum_v (csum_v, error))
- goto out;
- }
- csum_v = NULL;
- bloom = NULL;
-
- g_variant_iter_free (content_iter);
- g_variant_get_child (superindex, 3, "a(ayay)", &content_iter);
- while (g_variant_iter_loop (content_iter, "(@ay@ay)",
- &csum_v, &bloom))
- {
- if (!ostree_validate_structureof_csum_v (csum_v, error))
- goto out;
- }
- csum_v = NULL;
- bloom = NULL;
-
- ret = TRUE;
- out:
- if (content_iter)
- g_variant_iter_free (content_iter);
- return ret;
-}
*/
#define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE ("(a{sv}aya(say)sstayay)")
-/* Pack super index
- * s - OSTv0SUPERPACKINDEX
- * a{sv} - Metadata
- * a(ayay) - metadata packs (pack file checksum, bloom filter)
- * a(ayay) - data packs (pack file checksum, bloom filter)
- */
-#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay)a(ayay))")
-
-/* Pack index
- * s - OSTv0PACKINDEX
- * a{sv} - Metadata
- * a(yayt) - (objtype, checksum, offset into packfile)
- */
-#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(yayt))")
-
-typedef enum {
- OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
- OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0)
-} OstreePackFileEntryFlag;
-
-/* Data Pack files
- * s - OSTv0PACKDATAFILE
- * a{sv} - Metadata
- * t - number of entries
- *
- * Repeating pair of:
- * <padding to alignment of 8>
- * ( ayy(uuuusa(ayay))ay) ) - checksum, flags, file meta, data
- */
-#define OSTREE_PACK_DATA_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(ayy(uuuusa(ayay))ay)")
-
-/* Meta Pack files
- * s - OSTv0PACKMETAFILE
- * a{sv} - Metadata
- * t - number of entries
- *
- * Repeating pair of:
- * <padding to alignment of 8>
- * ( yayv ) - objtype, checksum, data
- */
-#define OSTREE_PACK_META_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yayv)")
-
const GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype);
gboolean ostree_validate_checksum_string (const char *sha256,
char *ostree_get_relative_archive_content_path (const char *checksum);
-char *ostree_get_pack_index_name (gboolean is_meta,
- const char *checksum);
-char *ostree_get_pack_data_name (gboolean is_meta,
- const char *checksum);
-
-char *ostree_get_relative_pack_index_path (gboolean is_meta,
- const char *checksum);
-char *ostree_get_relative_pack_data_path (gboolean is_meta,
- const char *checksum);
-
gboolean ostree_get_xattrs_for_file (GFile *f,
GVariant **out_xattrs,
GCancellable *cancellable,
GCancellable *cancellable,
GError **error);
-gboolean ostree_read_pack_entry_raw (guchar *pack_data,
- guint64 pack_len,
- guint64 object_offset,
- gboolean trusted,
- gboolean is_meta,
- GVariant **out_entry,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_parse_file_pack_entry (GVariant *pack_entry,
- GInputStream **out_input,
- GFileInfo **out_info,
- GVariant **out_xattrs,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_pack_index_search (GVariant *index,
- GVariant *csum_bytes,
- OstreeObjectType objtype,
- guint64 *out_offset);
-
/** VALIDATION **/
gboolean ostree_validate_structureof_objtype (guchar objtype,
gboolean ostree_validate_structureof_dirmeta (GVariant *index,
GError **error);
-gboolean ostree_validate_structureof_pack_index (GVariant *index,
- GError **error);
-
-gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex,
- GError **error);
-
#endif /* _OSTREE_REPO */
GFile *local_heads_dir;
GFile *remote_heads_dir;
GFile *objects_dir;
- GFile *pack_dir;
GFile *remote_cache_dir;
GFile *config_file;
#endif
GPtrArray *cached_meta_indexes;
GPtrArray *cached_content_indexes;
- GHashTable *cached_pack_index_mappings;
- GHashTable *cached_pack_data_mappings;
gboolean inited;
gboolean in_transaction;
repo_find_object (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
- gboolean lookup_all,
GFile **out_stored_path,
- char **out_pack_checksum,
- guint64 *out_pack_offset,
GCancellable *cancellable,
GError **error);
g_clear_object (&self->local_heads_dir);
g_clear_object (&self->remote_heads_dir);
g_clear_object (&self->objects_dir);
- g_clear_object (&self->pack_dir);
g_clear_object (&self->remote_cache_dir);
g_clear_object (&self->config_file);
if (self->loose_object_devino_hash)
g_key_file_free (self->config);
g_clear_pointer (&self->cached_meta_indexes, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&self->cached_content_indexes, (GDestroyNotify) g_ptr_array_unref);
- g_hash_table_destroy (self->cached_pack_index_mappings);
- g_hash_table_destroy (self->cached_pack_data_mappings);
g_mutex_clear (&self->cache_lock);
G_OBJECT_CLASS (ostree_repo_parent_class)->finalize (object);
self->remote_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/remotes");
self->objects_dir = g_file_get_child (self->repodir, "objects");
- self->pack_dir = g_file_get_child (self->objects_dir, "pack");
self->remote_cache_dir = g_file_get_child (self->repodir, "remote-cache");
self->config_file = g_file_get_child (self->repodir, "config");
ostree_repo_init (OstreeRepo *self)
{
g_mutex_init (&self->cache_lock);
- self->cached_pack_index_mappings = g_hash_table_new_full (g_str_hash, g_str_equal,
- g_free,
- (GDestroyNotify)g_variant_unref);
- self->cached_pack_data_mappings = g_hash_table_new_full (g_str_hash, g_str_equal,
- g_free,
- (GDestroyNotify)g_mapped_file_unref);
}
OstreeRepo*
}
typedef enum {
- OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED = (1<<0),
- OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<1)
+ OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<0)
} OstreeRepoStageFlags;
static gboolean
ot_lobj GFile *temp_file = NULL;
ot_lobj GFile *raw_temp_file = NULL;
ot_lobj GFile *stored_path = NULL;
- ot_lfree char *pack_checksum = NULL;
ot_lfree guchar *ret_csum = NULL;
ot_lobj OstreeChecksumInputStream *checksum_input = NULL;
+ gboolean have_obj;
GChecksum *checksum = NULL;
gboolean staged_raw_file = FALSE;
gboolean staged_archive_file = FALSE;
}
}
- if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED))
- {
- gboolean have_obj;
-
- if (!ostree_repo_has_object (self, objtype, actual_checksum, &have_obj,
- cancellable, error))
- goto out;
+ if (!ostree_repo_has_object (self, objtype, actual_checksum, &have_obj,
+ cancellable, error))
+ goto out;
- do_commit = !have_obj;
- }
- else
- do_commit = TRUE;
+ do_commit = !have_obj;
if (do_commit)
{
GError **error)
{
gboolean ret = FALSE;
- guint64 pack_offset;
ot_lobj GFile *stored_path = NULL;
- ot_lfree char *pack_checksum = NULL;
ot_lfree guchar *ret_csum = NULL;
g_return_val_if_fail (self->in_transaction, FALSE);
if (expected_checksum)
{
- if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED))
- {
- if (!repo_find_object (self, objtype, expected_checksum, FALSE,
- &stored_path, &pack_checksum, &pack_offset,
- cancellable, error))
- goto out;
- }
- else
- {
- if (!repo_find_object (self, objtype, expected_checksum, FALSE,
- &stored_path, NULL, NULL,
- cancellable, error))
- goto out;
- }
+ if (!repo_find_object (self, objtype, expected_checksum, &stored_path,
+ cancellable, error))
+ goto out;
}
- if (stored_path == NULL && pack_checksum == NULL)
+ if (stored_path == NULL)
{
if (!stage_object_internal (self, flags, objtype, input,
file_object_length, expected_checksum,
ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
- gboolean store_if_packed,
GInputStream *object_input,
GCancellable *cancellable,
GError **error)
{
int flags = 0;
- if (store_if_packed)
- flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED;
return stage_object (self, flags, objtype,
object_input, 0, checksum, NULL,
cancellable, error);
gboolean
ostree_repo_stage_file_object_trusted (OstreeRepo *self,
const char *checksum,
- gboolean store_if_packed,
GInputStream *object_input,
guint64 length,
GCancellable *cancellable,
GError **error)
{
int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
- if (store_if_packed)
- flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED;
return stage_object (self, flags, OSTREE_OBJECT_TYPE_FILE,
object_input, length, checksum, NULL,
cancellable, error);
if (!out)
goto out;
- g_hash_table_iter_init (&hash_iter, all_refs);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- const char *name = key;
- const char *sha256 = value;
-
- g_free (buf);
- buf = g_strdup_printf ("%s %s\n", sha256, name);
- if (!g_output_stream_write_all (out, buf, strlen (buf), &bytes_written, cancellable, error))
- goto out;
- }
-
- if (!g_output_stream_close (out, cancellable, error))
- goto out;
-
- ret = TRUE;
- out:
- return ret;
-}
-
-gboolean
-ostree_repo_write_ref (OstreeRepo *self,
- const char *remote,
- const char *name,
- const char *rev,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *dir = NULL;
-
- if (remote == NULL)
- dir = g_object_ref (self->local_heads_dir);
- else
- {
- dir = g_file_get_child (self->remote_heads_dir, remote);
-
- if (!ot_gfile_ensure_directory (dir, FALSE, error))
- goto out;
- }
-
- if (!write_checksum_file (dir, name, rev, error))
- goto out;
-
- if (self->mode == OSTREE_REPO_MODE_ARCHIVE)
- {
- if (!write_ref_summary (self, NULL, error))
- goto out;
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-gboolean
-ostree_repo_stage_commit (OstreeRepo *self,
- const char *branch,
- const char *parent,
- const char *subject,
- const char *body,
- GVariant *metadata,
- GVariant *related_objects,
- const char *root_contents_checksum,
- const char *root_metadata_checksum,
- char **out_commit,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lfree char *ret_commit = NULL;
- ot_lvariant GVariant *commit = NULL;
- ot_lfree guchar *commit_csum = NULL;
- GDateTime *now = NULL;
-
- g_return_val_if_fail (branch != NULL, FALSE);
- g_return_val_if_fail (subject != NULL, FALSE);
- g_return_val_if_fail (root_contents_checksum != NULL, FALSE);
- g_return_val_if_fail (root_metadata_checksum != NULL, FALSE);
-
- now = g_date_time_new_now_utc ();
- commit = g_variant_new ("(@a{sv}@ay@a(say)sst@ay@ay)",
- metadata ? metadata : create_empty_gvariant_dict (),
- parent ? ostree_checksum_to_bytes_v (parent) : ot_gvariant_new_bytearray (NULL, 0),
- related_objects ? related_objects : g_variant_new_array (G_VARIANT_TYPE ("(say)"), NULL, 0),
- subject, body ? body : "",
- GUINT64_TO_BE (g_date_time_to_unix (now)),
- ostree_checksum_to_bytes_v (root_contents_checksum),
- ostree_checksum_to_bytes_v (root_metadata_checksum));
- g_variant_ref_sink (commit);
- if (!stage_metadata_object (self, OSTREE_OBJECT_TYPE_COMMIT,
- commit, &commit_csum, cancellable, error))
- goto out;
-
- ret_commit = ostree_checksum_from_bytes (commit_csum);
-
- ret = TRUE;
- ot_transfer_out_value(out_commit, &ret_commit);
- out:
- if (now)
- g_date_time_unref (now);
- return ret;
-}
-
-static gboolean
-list_files_in_dir_matching (GFile *dir,
- const char *prefix,
- const char *suffix,
- GPtrArray **out_files,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- GError *temp_error = NULL;
- ot_lobj GFileEnumerator *enumerator = NULL;
- ot_lobj GFileInfo *file_info = NULL;
- ot_lptrarray GPtrArray *ret_files = NULL;
-
- g_return_val_if_fail (prefix != NULL || suffix != NULL, FALSE);
-
- ret_files = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
-
- enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- cancellable,
- error);
- if (!enumerator)
- goto out;
-
- while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL)
- {
- const char *name;
- guint32 type;
-
- name = g_file_info_get_attribute_byte_string (file_info, "standard::name");
- type = g_file_info_get_attribute_uint32 (file_info, "standard::type");
-
- if (type != G_FILE_TYPE_REGULAR)
- goto loop_next;
-
- if (prefix)
- {
- if (!g_str_has_prefix (name, prefix))
- goto loop_next;
- }
- if (suffix)
- {
- if (!g_str_has_suffix (name, suffix))
- goto loop_next;
- }
-
- g_ptr_array_add (ret_files, g_file_get_child (dir, name));
-
- loop_next:
- g_clear_object (&file_info);
- }
- if (temp_error != NULL)
- {
- g_propagate_error (error, temp_error);
- goto out;
- }
- if (!g_file_enumerator_close (enumerator, cancellable, error))
- goto out;
-
- ret = TRUE;
- ot_transfer_out_value (out_files, &ret_files);
- out:
- return ret;
-}
-
-static gboolean
-map_variant_file_check_header_string (GFile *path,
- const GVariantType *variant_type,
- const char *expected_header,
- gboolean trusted,
- GVariant **out_variant,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- const char *header;
- ot_lvariant GVariant *ret_variant = NULL;
-
- if (!ot_util_variant_map (path, variant_type, trusted, &ret_variant, error))
- goto out;
-
- g_variant_get_child (ret_variant, 0, "&s", &header);
-
- if (strcmp (header, expected_header) != 0)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid variant file '%s', expected header '%s'",
- ot_gfile_get_path_cached (path),
- expected_header);
- goto out;
- }
-
- ret = TRUE;
- ot_transfer_out_value (out_variant, &ret_variant);
- out:
- return ret;
-}
-
-
-static char *
-get_checksum_from_pack_name (const char *name)
-{
- const char *dash;
- const char *dot;
-
- dash = strchr (name, '-');
- g_assert (dash);
- dot = strrchr (name, '.');
- g_assert (dot);
-
- g_assert_cmpint (dot - (dash + 1), ==, 64);
-
- return g_strndup (dash + 1, 64);
-}
-
-static gboolean
-list_pack_indexes_from_dir (OstreeRepo *self,
- gboolean is_meta,
- GPtrArray **out_indexes,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- ot_lptrarray GPtrArray *index_files = NULL;
- ot_lptrarray GPtrArray *ret_indexes = NULL;
-
- if (!list_files_in_dir_matching (self->pack_dir,
- is_meta ? "ostmetapack-" : "ostdatapack-", ".index",
- &index_files,
- cancellable, error))
- goto out;
-
- ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
- for (i = 0; i < index_files->len; i++)
- {
- GFile *index_path = index_files->pdata[i];
- const char *basename = ot_gfile_get_basename_cached (index_path);
- g_ptr_array_add (ret_indexes, get_checksum_from_pack_name (basename));
- }
-
- ret = TRUE;
- ot_transfer_out_value (out_indexes, &ret_indexes);
- out:
- return ret;
-}
-
-static gboolean
-list_pack_checksums_from_superindex_file (GFile *superindex_path,
- GPtrArray **out_meta_indexes,
- GPtrArray **out_data_indexes,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- const char *magic;
- ot_lptrarray GPtrArray *ret_meta_indexes = NULL;
- ot_lptrarray GPtrArray *ret_data_indexes = NULL;
- ot_lvariant GVariant *superindex_variant = NULL;
- ot_lvariant GVariant *checksum = NULL;
- ot_lvariant GVariant *bloom = NULL;
- GVariantIter *meta_variant_iter = NULL;
- GVariantIter *data_variant_iter = NULL;
-
- if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
- TRUE, &superindex_variant, error))
- goto out;
-
- g_variant_get (superindex_variant, "(&s@a{sv}a(ayay)a(ayay))",
- &magic, NULL, &meta_variant_iter, &data_variant_iter);
-
- if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0)
- {
- g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid header in super pack index");
- goto out;
- }
-
- ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
- while (g_variant_iter_loop (meta_variant_iter, "(@ay@ay)",
- &checksum, &bloom))
- g_ptr_array_add (ret_meta_indexes, ostree_checksum_from_bytes_v (checksum));
- checksum = NULL;
- bloom = NULL;
-
- ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
- while (g_variant_iter_loop (data_variant_iter, "(@ay@ay)",
- &checksum, &bloom))
- g_ptr_array_add (ret_data_indexes, ostree_checksum_from_bytes_v (checksum));
- checksum = NULL;
- bloom = NULL;
-
- ret = TRUE;
- ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes);
- ot_transfer_out_value (out_data_indexes, &ret_data_indexes);
- out:
- if (meta_variant_iter)
- g_variant_iter_free (meta_variant_iter);
- if (data_variant_iter)
- g_variant_iter_free (data_variant_iter);
- return ret;
-}
-
-gboolean
-ostree_repo_list_pack_indexes (OstreeRepo *self,
- GPtrArray **out_meta_indexes,
- GPtrArray **out_data_indexes,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *superindex_path = NULL;
- ot_lptrarray GPtrArray *ret_meta_indexes = NULL;
- ot_lptrarray GPtrArray *ret_data_indexes = NULL;
-
- g_mutex_lock (&self->cache_lock);
- if (self->cached_meta_indexes)
- {
- ret_meta_indexes = g_ptr_array_ref (self->cached_meta_indexes);
- ret_data_indexes = g_ptr_array_ref (self->cached_content_indexes);
- }
- else
- {
- superindex_path = g_file_get_child (self->pack_dir, "index");
-
- if (g_file_query_exists (superindex_path, cancellable))
- {
- if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_meta_indexes,
- &ret_data_indexes,
- cancellable, error))
- goto out;
- }
- else
- {
- ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
- ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
- }
-
- self->cached_meta_indexes = g_ptr_array_ref (ret_meta_indexes);
- self->cached_content_indexes = g_ptr_array_ref (ret_data_indexes);
- }
-
- ret = TRUE;
- ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes);
- ot_transfer_out_value (out_data_indexes, &ret_data_indexes);
- out:
- g_mutex_unlock (&self->cache_lock);
- return ret;
-}
-
-static gboolean
-create_index_bloom (OstreeRepo *self,
- const char *pack_checksum,
- GVariant **out_bloom,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lvariant GVariant *ret_bloom;
-
- /* TODO - define and compute bloom filter */
-
- ret_bloom = ot_gvariant_new_bytearray (NULL, 0);
- g_variant_ref_sink (ret_bloom);
-
- ret = TRUE;
- ot_transfer_out_value (out_bloom, &ret_bloom);
- /* out: */
- return ret;
-}
-
-static gboolean
-append_index_builder (OstreeRepo *self,
- GPtrArray *indexes,
- GVariantBuilder *builder,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
-
- for (i = 0; i < indexes->len; i++)
- {
- const char *pack_checksum = indexes->pdata[i];
- ot_lvariant GVariant *bloom = NULL;
-
- if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error))
- goto out;
-
- g_variant_builder_add (builder,
- "(@ay@ay)",
- ostree_checksum_to_bytes_v (pack_checksum),
- bloom);
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-/**
- * Regenerate the pack superindex file based on the set of pack
- * indexes currently in the filesystem.
- */
-gboolean
-ostree_repo_regenerate_pack_index (OstreeRepo *self,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *superindex_path = NULL;
- ot_lptrarray GPtrArray *pack_indexes = NULL;
- ot_lvariant GVariant *superindex_variant = NULL;
- GVariantBuilder *meta_index_content_builder = NULL;
- GVariantBuilder *data_index_content_builder = NULL;
-
- g_clear_pointer (&self->cached_meta_indexes, (GDestroyNotify) g_ptr_array_unref);
- g_clear_pointer (&self->cached_content_indexes, (GDestroyNotify) g_ptr_array_unref);
-
- superindex_path = g_file_get_child (self->pack_dir, "index");
-
- g_clear_pointer (&pack_indexes, (GDestroyNotify) g_ptr_array_unref);
- if (!list_pack_indexes_from_dir (self, TRUE, &pack_indexes,
- cancellable, error))
- goto out;
- meta_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)"));
- if (!append_index_builder (self, pack_indexes, meta_index_content_builder,
- cancellable, error))
- goto out;
-
- g_clear_pointer (&pack_indexes, (GDestroyNotify) g_ptr_array_unref);
- if (!list_pack_indexes_from_dir (self, FALSE, &pack_indexes,
- cancellable, error))
- goto out;
- data_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)"));
- if (!append_index_builder (self, pack_indexes, data_index_content_builder,
- cancellable, error))
- goto out;
-
- superindex_variant = g_variant_new ("(s@a{sv}@a(ayay)@a(ayay))",
- "OSTv0SUPERPACKINDEX",
- g_variant_new_array (G_VARIANT_TYPE ("{sv}"),
- NULL, 0),
- g_variant_builder_end (meta_index_content_builder),
- g_variant_builder_end (data_index_content_builder));
- g_variant_ref_sink (superindex_variant);
-
- if (!ot_util_variant_save (superindex_path, superindex_variant,
- cancellable, error))
- goto out;
-
- ret = TRUE;
- out:
- if (meta_index_content_builder)
- g_variant_builder_unref (meta_index_content_builder);
- if (data_index_content_builder)
- g_variant_builder_unref (data_index_content_builder);
- return ret;
-}
-
-static GFile *
-get_pack_index_path (GFile *parent,
- gboolean is_meta,
- const char *checksum)
-{
- char *path = ostree_get_pack_index_name (is_meta, checksum);
- GFile *ret = g_file_resolve_relative_path (parent, path);
- g_free (path);
- return ret;
-}
-
-static GFile *
-get_pack_data_path (GFile *parent,
- gboolean is_meta,
- const char *checksum)
-{
- char *path = ostree_get_pack_data_name (is_meta, checksum);
- GFile *ret = g_file_resolve_relative_path (parent, path);
- g_free (path);
- return ret;
-}
-
-gboolean
-ostree_repo_add_pack_file (OstreeRepo *self,
- const char *pack_checksum,
- gboolean is_meta,
- GFile *index_path,
- GFile *data_path,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *pack_index_path = NULL;
- ot_lobj GFile *pack_data_path = NULL;
-
- if (!ot_gfile_ensure_directory (self->pack_dir, FALSE, error))
- goto out;
-
- pack_data_path = get_pack_data_path (self->pack_dir, is_meta, pack_checksum);
- if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error))
- goto out;
-
- pack_index_path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum);
- if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error))
- goto out;
-
- ret = TRUE;
- out:
- return ret;
-}
-
-static gboolean
-ensure_remote_cache_dir (OstreeRepo *self,
- const char *remote_name,
- GFile **out_cache_dir,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *ret_cache_dir = NULL;
-
- ret_cache_dir = g_file_get_child (self->remote_cache_dir, remote_name);
-
- if (!ot_gfile_ensure_directory (ret_cache_dir, FALSE, error))
- goto out;
-
- ret = TRUE;
- ot_transfer_out_value (out_cache_dir, &ret_cache_dir);
- out:
- return ret;
-}
-
-static gboolean
-delete_no_longer_referenced (OstreeRepo *self,
- GFile *cache_path,
- const char *prefix,
- const char *suffix,
- GHashTable *new_files,
- GPtrArray *inout_cached,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- ot_lptrarray GPtrArray *current_files = NULL;
- ot_lfree char *pack_checksum = NULL;
-
- if (!list_files_in_dir_matching (cache_path,
- prefix, suffix,
- ¤t_files,
- cancellable, error))
- goto out;
- for (i = 0; i < current_files->len; i++)
- {
- GFile *file = current_files->pdata[i];
-
- g_free (pack_checksum);
- pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (file));
-
- if (!g_hash_table_lookup (new_files, pack_checksum))
- {
- if (!ot_gfile_unlink (file, cancellable, error))
- goto out;
- }
-
- if (inout_cached)
- {
- g_ptr_array_add (inout_cached, pack_checksum);
- pack_checksum = NULL; /* transfer ownership */
- }
- }
- ret = TRUE;
- out:
- return ret;
-}
-
-static void
-gather_uncached (GHashTable *new_files,
- GPtrArray *cached,
- GPtrArray *inout_uncached)
-{
- guint i;
- GHashTableIter hash_iter;
- gpointer key, value;
-
- g_hash_table_iter_init (&hash_iter, new_files);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- const char *cur_pack_checksum = key;
- gboolean found = FALSE;
-
- for (i = 0; i < cached->len; i++)
- {
- const char *checksum = cached->pdata[i];
- if (strcmp (cur_pack_checksum, checksum) == 0)
- {
- found = TRUE;
- break;
- }
- }
-
- if (!found)
- g_ptr_array_add (inout_uncached, g_strdup (cur_pack_checksum));
- }
-}
-
-/**
- * Take a pack superindex file @superindex_path, and clean up any
- * no-longer-referenced pack files in the lookaside cache for
- * @remote_name. The updated index file will also be saved into the
- * cache.
- *
- * Upon successful return, @out_cached_indexes will hold checksum
- * strings for indexes which are already in the cache, and
- * @out_uncached_indexes will hold strings for those which are not.
- */
-gboolean
-ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
- const char *remote_name,
- GFile *superindex_path,
- GPtrArray **out_cached_meta_indexes,
- GPtrArray **out_cached_data_indexes,
- GPtrArray **out_uncached_meta_indexes,
- GPtrArray **out_uncached_data_indexes,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lvariant GVariant *superindex_variant = NULL;
- ot_lobj GFile *cache_path = NULL;
- ot_lobj GFile *superindex_cache_path = NULL;
- ot_lptrarray GPtrArray *meta_index_files = NULL;
- ot_lptrarray GPtrArray *data_index_files = NULL;
- ot_lptrarray GPtrArray *meta_data_files = NULL;
- ot_lptrarray GPtrArray *data_data_files = NULL;
- ot_lhash GHashTable *new_pack_meta_indexes = NULL;
- ot_lhash GHashTable *new_pack_data_indexes = NULL;
- ot_lptrarray GPtrArray *ret_cached_meta_indexes = NULL;
- ot_lptrarray GPtrArray *ret_cached_data_indexes = NULL;
- ot_lptrarray GPtrArray *ret_uncached_meta_indexes = NULL;
- ot_lptrarray GPtrArray *ret_uncached_data_indexes = NULL;
- ot_lvariant GVariant *csum_bytes = NULL;
- ot_lvariant GVariant *bloom = NULL;
- ot_lfree char *pack_checksum = NULL;
- GVariantIter *superindex_contents_iter = NULL;
-
- if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error))
- goto out;
-
- ret_cached_meta_indexes = g_ptr_array_new_with_free_func (g_free);
- ret_cached_data_indexes = g_ptr_array_new_with_free_func (g_free);
- ret_uncached_meta_indexes = g_ptr_array_new_with_free_func (g_free);
- ret_uncached_data_indexes = g_ptr_array_new_with_free_func (g_free);
-
- if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
- FALSE, &superindex_variant, error))
- goto out;
-
- if (!ostree_validate_structureof_pack_superindex (superindex_variant, error))
- goto out;
-
- new_pack_meta_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
- new_pack_data_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
-
- g_variant_get_child (superindex_variant, 2, "a(ayay)",
- &superindex_contents_iter);
- while (g_variant_iter_loop (superindex_contents_iter,
- "(@ay@ay)", &csum_bytes, &bloom))
- {
- pack_checksum = ostree_checksum_from_bytes_v (csum_bytes);
- g_hash_table_insert (new_pack_meta_indexes, pack_checksum, pack_checksum);
- pack_checksum = NULL; /* transfer ownership */
- }
- g_variant_iter_free (superindex_contents_iter);
-
- g_variant_get_child (superindex_variant, 3, "a(ayay)",
- &superindex_contents_iter);
- while (g_variant_iter_loop (superindex_contents_iter,
- "(@ay@ay)", &csum_bytes, &bloom))
- {
- pack_checksum = ostree_checksum_from_bytes_v (csum_bytes);
- g_hash_table_insert (new_pack_data_indexes, pack_checksum, pack_checksum);
- pack_checksum = NULL; /* transfer ownership */
- }
-
- if (!delete_no_longer_referenced (self, cache_path,
- "ostmetapack-", ".index",
- new_pack_meta_indexes,
- ret_cached_meta_indexes,
- cancellable, error))
- goto out;
-
- if (!delete_no_longer_referenced (self, cache_path,
- "ostdatapack-", ".index",
- new_pack_data_indexes,
- ret_cached_data_indexes,
- cancellable, error))
- goto out;
-
- gather_uncached (new_pack_meta_indexes, ret_cached_meta_indexes, ret_uncached_meta_indexes);
- gather_uncached (new_pack_data_indexes, ret_cached_data_indexes, ret_uncached_data_indexes);
-
- superindex_cache_path = g_file_get_child (cache_path, "index");
- if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error))
- goto out;
-
- /* Now also delete stale pack files */
-
- if (!delete_no_longer_referenced (self, cache_path,
- "ostmetapack-", ".data",
- new_pack_meta_indexes, NULL,
- cancellable, error))
- goto out;
- if (!delete_no_longer_referenced (self, cache_path,
- "ostdatapack-", ".data",
- new_pack_data_indexes, NULL,
- cancellable, error))
- goto out;
-
- ret = TRUE;
- ot_transfer_out_value (out_cached_meta_indexes, &ret_cached_meta_indexes);
- ot_transfer_out_value (out_cached_data_indexes, &ret_cached_data_indexes);
- ot_transfer_out_value (out_uncached_meta_indexes, &ret_uncached_meta_indexes);
- ot_transfer_out_value (out_uncached_data_indexes, &ret_uncached_data_indexes);
- out:
- if (superindex_contents_iter)
- g_variant_iter_free (superindex_contents_iter);
- return ret;
-}
-
-gboolean
-ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self,
- const char *remote_name,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- ot_lobj GFile *cache_path = NULL;
- ot_lptrarray GPtrArray *data_files = NULL;
-
- if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error))
- goto out;
-
- if (!list_files_in_dir_matching (cache_path,
- "ostmetapack-", ".data",
- &data_files,
- cancellable, error))
- goto out;
- for (i = 0; i < data_files->len; i++)
- {
- GFile *data_file = data_files->pdata[i];
-
- if (!ot_gfile_unlink (data_file, cancellable, error))
- goto out;
- }
-
- g_clear_pointer (&data_files, (GDestroyNotify) g_ptr_array_unref);
- if (!list_files_in_dir_matching (cache_path,
- "ostdatapack-", ".data",
- &data_files,
- cancellable, error))
- goto out;
- for (i = 0; i < data_files->len; i++)
- {
- GFile *data_file = data_files->pdata[i];
-
- if (!ot_gfile_unlink (data_file, cancellable, error))
- goto out;
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-/**
- * Load the index for pack @pack_checksum from cache directory for
- * @remote_name.
- */
-gboolean
-ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
- const char *remote_name,
- const char *pack_checksum,
- gboolean is_meta,
- GVariant **out_variant,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lvariant GVariant *ret_variant = NULL;
- ot_lobj GFile *cache_dir = NULL;
- ot_lobj GFile *cached_pack_path = NULL;
-
- if (!ensure_remote_cache_dir (self, remote_name, &cache_dir,
- cancellable, error))
- goto out;
-
- cached_pack_path = get_pack_index_path (cache_dir, is_meta, pack_checksum);
- if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT,
- FALSE, &ret_variant, error))
- goto out;
-
- ret = TRUE;
- ot_transfer_out_value (out_variant, &ret_variant);
- out:
- return ret;
-}
-
-/**
- * The variable @cached_path should refer to a file containing a pack
- * index. It will be validated and added to the cache directory for
- * @remote_name.
- */
-gboolean
-ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
- const char *remote_name,
- const char *pack_checksum,
- gboolean is_meta,
- GFile *cached_path,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *cachedir = NULL;
- ot_lobj GFile *target_path = NULL;
- ot_lvariant GVariant *input_index_variant = NULL;
- ot_lvariant GVariant *output_index_variant = NULL;
-
- if (!map_variant_file_check_header_string (cached_path,
- OSTREE_PACK_INDEX_VARIANT_FORMAT,
- "OSTv0PACKINDEX",
- FALSE, &input_index_variant,
- cancellable, error))
- goto out;
-
- if (!ostree_validate_structureof_pack_index (input_index_variant, error))
- goto out;
-
- output_index_variant = g_variant_get_normal_form (input_index_variant);
-
- if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
- goto out;
-
- target_path = get_pack_index_path (cachedir, is_meta, pack_checksum);
- if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error))
+ g_hash_table_iter_init (&hash_iter, all_refs);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ const char *name = key;
+ const char *sha256 = value;
+
+ g_free (buf);
+ buf = g_strdup_printf ("%s %s\n", sha256, name);
+ if (!g_output_stream_write_all (out, buf, strlen (buf), &bytes_written, cancellable, error))
+ goto out;
+ }
+
+ if (!g_output_stream_close (out, cancellable, error))
goto out;
ret = TRUE;
return ret;
}
-/**
- * Check for availability of the pack index pointing to @pack_checksum
- * in the lookaside cache for @remote_name. If not found, then the
- * output parameter @out_cached_path will be %NULL.
- */
-gboolean
-ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
- const char *remote_name,
- const char *pack_checksum,
- gboolean is_meta,
- GFile **out_cached_path,
- GCancellable *cancellable,
- GError **error)
+gboolean
+ostree_repo_write_ref (OstreeRepo *self,
+ const char *remote,
+ const char *name,
+ const char *rev,
+ GError **error)
{
gboolean ret = FALSE;
- ot_lobj GFile *cache_dir = NULL;
- ot_lobj GFile *cached_pack_path = NULL;
- ot_lobj GFile *ret_cached_path = NULL;
+ ot_lobj GFile *dir = NULL;
+
+ if (remote == NULL)
+ dir = g_object_ref (self->local_heads_dir);
+ else
+ {
+ dir = g_file_get_child (self->remote_heads_dir, remote);
+
+ if (!ot_gfile_ensure_directory (dir, FALSE, error))
+ goto out;
+ }
- if (!ensure_remote_cache_dir (self, remote_name, &cache_dir,
- cancellable, error))
+ if (!write_checksum_file (dir, name, rev, error))
goto out;
- cached_pack_path = get_pack_data_path (cache_dir, is_meta, pack_checksum);
- if (g_file_query_exists (cached_pack_path, cancellable))
+ if (self->mode == OSTREE_REPO_MODE_ARCHIVE)
{
- ret_cached_path = cached_pack_path;
- cached_pack_path = NULL;
+ if (!write_ref_summary (self, NULL, error))
+ goto out;
}
ret = TRUE;
- ot_transfer_out_value (out_cached_path, &ret_cached_path);
out:
return ret;
}
-/**
- * Add file @cached_path into the cache for given @remote_name. If
- * @cached_path is %NULL, delete the cached pack data (if any).
- *
- * <note>
- * This unlinks @cached_path.
- * </note>
- */
gboolean
-ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
- const char *remote_name,
- const char *pack_checksum,
- gboolean is_meta,
- GFile *cached_path,
- GCancellable *cancellable,
- GError **error)
+ostree_repo_stage_commit (OstreeRepo *self,
+ const char *branch,
+ const char *parent,
+ const char *subject,
+ const char *body,
+ GVariant *metadata,
+ GVariant *related_objects,
+ const char *root_contents_checksum,
+ const char *root_metadata_checksum,
+ char **out_commit,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- ot_lobj GFile *cachedir = NULL;
- ot_lobj GFile *target_path = NULL;
+ ot_lfree char *ret_commit = NULL;
+ ot_lvariant GVariant *commit = NULL;
+ ot_lfree guchar *commit_csum = NULL;
+ GDateTime *now = NULL;
- if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
+ g_return_val_if_fail (branch != NULL, FALSE);
+ g_return_val_if_fail (subject != NULL, FALSE);
+ g_return_val_if_fail (root_contents_checksum != NULL, FALSE);
+ g_return_val_if_fail (root_metadata_checksum != NULL, FALSE);
+
+ now = g_date_time_new_now_utc ();
+ commit = g_variant_new ("(@a{sv}@ay@a(say)sst@ay@ay)",
+ metadata ? metadata : create_empty_gvariant_dict (),
+ parent ? ostree_checksum_to_bytes_v (parent) : ot_gvariant_new_bytearray (NULL, 0),
+ related_objects ? related_objects : g_variant_new_array (G_VARIANT_TYPE ("(say)"), NULL, 0),
+ subject, body ? body : "",
+ GUINT64_TO_BE (g_date_time_to_unix (now)),
+ ostree_checksum_to_bytes_v (root_contents_checksum),
+ ostree_checksum_to_bytes_v (root_metadata_checksum));
+ g_variant_ref_sink (commit);
+ if (!stage_metadata_object (self, OSTREE_OBJECT_TYPE_COMMIT,
+ commit, &commit_csum, cancellable, error))
goto out;
- target_path = get_pack_data_path (cachedir, is_meta, pack_checksum);
- if (cached_path)
- {
- if (!ot_gfile_rename (cached_path, target_path, cancellable, error))
- goto out;
- }
- else
- {
- (void) ot_gfile_unlink (target_path, cancellable, NULL);
- }
+ ret_commit = ostree_checksum_from_bytes (commit_csum);
ret = TRUE;
+ ot_transfer_out_value(out_commit, &ret_commit);
out:
+ if (now)
+ g_date_time_unref (now);
return ret;
}
return ret;
}
-gboolean
-ostree_repo_load_pack_index (OstreeRepo *self,
- const char *pack_checksum,
- gboolean is_meta,
- GVariant **out_variant,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lvariant GVariant *ret_variant = NULL;
- ot_lobj GFile *path = NULL;
-
- g_mutex_lock (&self->cache_lock);
-
- ret_variant = g_hash_table_lookup (self->cached_pack_index_mappings, pack_checksum);
- if (ret_variant)
- {
- g_variant_ref (ret_variant);
- }
- else
- {
- path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum);
- if (!map_variant_file_check_header_string (path,
- OSTREE_PACK_INDEX_VARIANT_FORMAT,
- "OSTv0PACKINDEX", TRUE,
- &ret_variant,
- cancellable, error))
- goto out;
- g_hash_table_insert (self->cached_pack_index_mappings, g_strdup (pack_checksum),
- g_variant_ref (ret_variant));
- }
-
- ret = TRUE;
- ot_transfer_out_value (out_variant, &ret_variant);
- out:
- g_mutex_unlock (&self->cache_lock);
- return ret;
-}
-
-/**
- * @sha256: Checksum of pack file
- * @out_data: (out): Pointer to pack file data
- *
- * Ensure that the given pack file is mapped into
- * memory.
- */
-gboolean
-ostree_repo_map_pack_file (OstreeRepo *self,
- const char *pack_checksum,
- gboolean is_meta,
- guchar **out_data,
- guint64 *out_len,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- gpointer ret_data;
- guint64 ret_len;
- GMappedFile *map = NULL;
- ot_lobj GFile *path = NULL;
-
- g_mutex_lock (&self->cache_lock);
-
- map = g_hash_table_lookup (self->cached_pack_data_mappings, pack_checksum);
- if (map == NULL)
- {
- path = get_pack_data_path (self->pack_dir, is_meta, pack_checksum);
-
- map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error);
- if (!map)
- goto out;
-
- g_hash_table_insert (self->cached_pack_data_mappings, g_strdup (pack_checksum), map);
- ret_data = g_mapped_file_get_contents (map);
- }
-
- ret_data = g_mapped_file_get_contents (map);
- ret_len = (guint64)g_mapped_file_get_length (map);
-
- ret = TRUE;
- if (out_data)
- *out_data = ret_data;
- if (out_len)
- *out_len = ret_len;
- out:
- g_mutex_unlock (&self->cache_lock);
- return ret;
-}
-
gboolean
ostree_repo_load_file (OstreeRepo *self,
const char *checksum,
GError **error)
{
gboolean ret = FALSE;
- guchar *pack_data;
- guint64 pack_len;
- guint64 pack_offset;
- ot_lvariant GVariant *packed_object = NULL;
ot_lvariant GVariant *file_data = NULL;
ot_lobj GFile *loose_path = NULL;
ot_lobj GFileInfo *content_loose_info = NULL;
- ot_lfree char *pack_checksum = NULL;
ot_lobj GInputStream *ret_input = NULL;
ot_lobj GFileInfo *ret_file_info = NULL;
ot_lvariant GVariant *ret_xattrs = NULL;
- if (!repo_find_object (self, OSTREE_OBJECT_TYPE_FILE,
- checksum, FALSE, &loose_path,
- &pack_checksum, &pack_offset,
+ if (!repo_find_object (self, OSTREE_OBJECT_TYPE_FILE, checksum, &loose_path,
cancellable, error))
goto out;
}
}
}
- else if (pack_checksum)
- {
- if (!ostree_repo_map_pack_file (self, pack_checksum, FALSE,
- &pack_data, &pack_len,
- cancellable, error))
- goto out;
-
- if (!ostree_read_pack_entry_raw (pack_data, pack_len,
- pack_offset, TRUE, FALSE,
- &packed_object, cancellable, error))
- goto out;
-
- if (!ostree_parse_file_pack_entry (packed_object,
- out_input ? &ret_input : NULL,
- out_file_info ? &ret_file_info : NULL,
- out_xattrs ? &ret_xattrs : NULL,
- cancellable, error))
- goto out;
- }
else if (self->parent_repo)
{
if (!ostree_repo_load_file (self->parent_repo, checksum,
return ret;
}
-static gboolean
-list_objects_in_index (OstreeRepo *self,
- const char *pack_checksum,
- gboolean is_meta,
- GHashTable *inout_objects,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint8 objtype_u8;
- guint64 offset;
- ot_lobj GFile *index_path = NULL;
- ot_lvariant GVariant *index_variant = NULL;
- ot_lvariant GVariant *contents = NULL;
- ot_lvariant GVariant *csum_bytes = NULL;
- ot_lfree char *checksum = NULL;
- GVariantIter content_iter;
-
- index_path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum);
-
- if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta,
- &index_variant, cancellable, error))
- goto out;
-
- contents = g_variant_get_child_value (index_variant, 2);
- g_variant_iter_init (&content_iter, contents);
-
- while (g_variant_iter_loop (&content_iter, "(y@ayt)", &objtype_u8, &csum_bytes, &offset))
- {
- GVariant *obj_key;
- GVariant *objdata;
- OstreeObjectType objtype;
- GVariantBuilder pack_contents_builder;
- gboolean is_loose;
-
- objtype = (OstreeObjectType) objtype_u8;
- offset = GUINT64_FROM_BE (offset);
-
- g_variant_builder_init (&pack_contents_builder,
- G_VARIANT_TYPE_STRING_ARRAY);
-
- g_free (checksum);
- checksum = ostree_checksum_from_bytes_v (csum_bytes);
- obj_key = ostree_object_name_serialize (checksum, objtype);
- ot_util_variant_take_ref (obj_key);
-
- objdata = g_hash_table_lookup (inout_objects, obj_key);
- if (!objdata)
- {
- is_loose = FALSE;
- }
- else
- {
- GVariantIter *current_packs_iter;
- const char *current_pack_checksum;
-
- g_variant_get (objdata, "(bas)", &is_loose, ¤t_packs_iter);
-
- while (g_variant_iter_loop (current_packs_iter, "&s", ¤t_pack_checksum))
- {
- g_variant_builder_add (&pack_contents_builder, "s", current_pack_checksum);
- }
- g_variant_iter_free (current_packs_iter);
- }
- g_variant_builder_add (&pack_contents_builder, "s", pack_checksum);
- objdata = g_variant_new ("(b@as)", is_loose,
- g_variant_builder_end (&pack_contents_builder));
- g_variant_ref_sink (objdata);
- g_hash_table_replace (inout_objects, obj_key, objdata);
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-static gboolean
-list_packed_objects (OstreeRepo *self,
- GHashTable *inout_objects,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- ot_lptrarray GPtrArray *meta_index_checksums = NULL;
- ot_lptrarray GPtrArray *data_index_checksums = NULL;
-
- if (!ostree_repo_list_pack_indexes (self, &meta_index_checksums, &data_index_checksums,
- cancellable, error))
- goto out;
-
- for (i = 0; i < meta_index_checksums->len; i++)
- {
- const char *checksum = meta_index_checksums->pdata[i];
- if (!list_objects_in_index (self, checksum, TRUE, inout_objects, cancellable, error))
- goto out;
- }
-
- for (i = 0; i < data_index_checksums->len; i++)
- {
- const char *checksum = data_index_checksums->pdata[i];
- if (!list_objects_in_index (self, checksum, FALSE, inout_objects, cancellable, error))
- goto out;
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-static gboolean
-find_object_in_packs (OstreeRepo *self,
- const char *checksum,
- OstreeObjectType objtype,
- char **out_pack_checksum,
- guint64 *out_pack_offset,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- guint64 ret_pack_offset = 0;
- gboolean is_meta;
- ot_lptrarray GPtrArray *index_checksums = NULL;
- ot_lfree char *ret_pack_checksum = NULL;
- ot_lvariant GVariant *csum_bytes = NULL;
- ot_lvariant GVariant *index_variant = NULL;
-
- csum_bytes = ostree_checksum_to_bytes_v (checksum);
-
- is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
-
- if (is_meta)
- {
- if (!ostree_repo_list_pack_indexes (self, &index_checksums, NULL,
- cancellable, error))
- goto out;
- }
- else
- {
- if (!ostree_repo_list_pack_indexes (self, NULL, &index_checksums,
- cancellable, error))
- goto out;
- }
-
- for (i = 0; i < index_checksums->len; i++)
- {
- const char *pack_checksum = index_checksums->pdata[i];
- guint64 offset;
-
- g_clear_pointer (&index_variant, (GDestroyNotify) g_variant_unref);
- if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, &index_variant,
- cancellable, error))
- goto out;
-
- if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset))
- continue;
-
- ret_pack_checksum = g_strdup (pack_checksum);
- ret_pack_offset = offset;
- break;
- }
-
- ret = TRUE;
- ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
- if (out_pack_offset)
- *out_pack_offset = ret_pack_offset;
- out:
- return ret;
-}
-
static gboolean
repo_find_object (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
- gboolean lookup_all,
GFile **out_stored_path,
- char **out_pack_checksum,
- guint64 *out_pack_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
- guint64 ret_pack_offset = 0;
struct stat stbuf;
ot_lobj GFile *object_path = NULL;
ot_lobj GFile *ret_stored_path = NULL;
- ot_lfree char *ret_pack_checksum = NULL;
-
- /* Look up metadata in packs first, but content loose first. We
- * want to find loose content since that's preferable for
- * hardlinking scenarios.
- *
- * Metadata is much more efficient packed.
- */
- if (OSTREE_OBJECT_TYPE_IS_META (objtype))
- {
- if (out_pack_checksum)
- {
- if (!find_object_in_packs (self, checksum, objtype,
- &ret_pack_checksum, &ret_pack_offset,
- cancellable, error))
- goto out;
- }
- if (!ret_pack_checksum || lookup_all)
- {
- object_path = ostree_repo_get_object_path (self, checksum, objtype);
+
+ object_path = ostree_repo_get_object_path (self, checksum, objtype);
- if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
- {
- ret_stored_path = object_path;
- object_path = NULL;
- }
- else
- {
- g_clear_object (&object_path);
- }
- }
+ if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
+ {
+ ret_stored_path = object_path;
+ object_path = NULL; /* Transfer ownership */
}
- else
+ else if (errno != ENOENT)
{
- if (out_stored_path)
- {
- object_path = ostree_repo_get_object_path (self, checksum, objtype);
-
- if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
- {
- ret_stored_path = object_path;
- object_path = NULL;
- }
- else
- {
- g_clear_object (&object_path);
- }
- }
- if (!ret_stored_path || lookup_all)
- {
- if (out_pack_checksum)
- {
- if (!find_object_in_packs (self, checksum, objtype,
- &ret_pack_checksum, &ret_pack_offset,
- cancellable, error))
- goto out;
- }
- }
+ ot_util_set_error_from_errno (error, errno);
+ goto out;
}
-
+
ret = TRUE;
ot_transfer_out_value (out_stored_path, &ret_stored_path);
- ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
- if (out_pack_offset)
- *out_pack_offset = ret_pack_offset;
out:
return ret;
}
gboolean ret = FALSE;
gboolean ret_have_object;
ot_lobj GFile *loose_path = NULL;
- ot_lfree char *pack_checksum = NULL;
- if (!repo_find_object (self, objtype, checksum, FALSE,
- &loose_path,
- &pack_checksum, NULL,
+ if (!repo_find_object (self, objtype, checksum, &loose_path,
cancellable, error))
goto out;
- ret_have_object = (loose_path != NULL) || (pack_checksum != NULL);
+ ret_have_object = (loose_path != NULL);
if (!ret_have_object && self->parent_repo)
{
GError **error)
{
gboolean ret = FALSE;
- guchar *pack_data;
- guint64 pack_len;
- guint64 object_offset;
GCancellable *cancellable = NULL;
ot_lobj GFile *object_path = NULL;
- ot_lvariant GVariant *packed_object = NULL;
ot_lvariant GVariant *ret_variant = NULL;
- ot_lfree char *pack_checksum = NULL;
g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE);
- if (!repo_find_object (self, objtype, sha256, FALSE,
- &object_path, &pack_checksum, &object_offset,
+ if (!repo_find_object (self, objtype, sha256, &object_path,
cancellable, error))
goto out;
- if (pack_checksum != NULL)
- {
- if (!ostree_repo_map_pack_file (self, pack_checksum, TRUE, &pack_data, &pack_len,
- cancellable, error))
- goto out;
-
- if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset,
- TRUE, TRUE, &packed_object, cancellable, error))
- goto out;
-
- g_variant_get_child (packed_object, 2, "v", &ret_variant);
- }
- else if (object_path != NULL)
+ if (object_path != NULL)
{
if (!ot_util_variant_map (object_path, ostree_metadata_variant_type (objtype),
TRUE, &ret_variant, error))
if (flags & OSTREE_REPO_LIST_OBJECTS_PACKED)
{
- if (!list_packed_objects (self, ret_objects, cancellable, error))
- goto out;
- if (self->parent_repo)
- {
- if (!list_packed_objects (self->parent_repo, ret_objects, cancellable, error))
- goto out;
- }
+ /* Nothing for now... */
}
ret = TRUE;
gboolean ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
- gboolean store_if_packed,
GInputStream *content,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_stage_file_object_trusted (OstreeRepo *self,
const char *checksum,
- gboolean store_if_packed,
GInputStream *content,
guint64 content_length,
GCancellable *cancellable,
GVariant **out_variant,
GError **error);
-gboolean ostree_repo_load_pack_index (OstreeRepo *self,
- const char *pack_checksum,
- gboolean is_meta,
- GVariant **out_variant,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_load_pack_data (OstreeRepo *self,
- const char *pack_checksum,
- guchar **out_data,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_map_pack_file (OstreeRepo *self,
- const char *sha256,
- gboolean is_meta,
- guchar **out_data,
- guint64 *out_len,
- GCancellable *cancellable,
- GError **error);
-
gboolean ostree_repo_load_file (OstreeRepo *self,
const char *entry_sha256,
GInputStream **out_input,
GCancellable *cancellable,
GError **error);
-gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_add_pack_file (OstreeRepo *self,
- const char *checksum,
- gboolean is_meta,
- GFile *pack_index_path,
- GFile *pack_data_path,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
- const char *remote_name,
- GFile *superindex_path,
- GPtrArray **out_cached_meta_indexes,
- GPtrArray **out_cached_data_indexes,
- GPtrArray **out_uncached_meta_indexes,
- GPtrArray **out_uncached_data_indexes,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self,
- const char *remote_name,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
- const char *remote_name,
- const char *pack_checksum,
- gboolean is_meta,
- GVariant **out_variant,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
- const char *remote_name,
- const char *pack_checksum,
- gboolean is_meta,
- GFile *cached_path,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
- const char *remote_name,
- const char *pack_checksum,
- gboolean is_meta,
- GFile **out_cached_path,
- GCancellable *cancellable,
- GError **error);
-
-gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
- const char *remote_name,
- const char *pack_checksum,
- gboolean is_meta,
- GFile *cached_path,
- GCancellable *cancellable,
- GError **error);
-
typedef enum {
OSTREE_REPO_CHECKOUT_MODE_NONE = 0,
OSTREE_REPO_CHECKOUT_MODE_USER = 1
GCancellable *cancellable,
GError **error);
-gboolean ostree_repo_list_pack_indexes (OstreeRepo *self,
- GPtrArray **out_meta_indexes,
- GPtrArray **out_data_indexes,
- GCancellable *cancellable,
- GError **error);
-
G_END_DECLS
#endif /* _OSTREE_REPO */
{ "init", ostree_builtin_init, 0 },
{ "log", ostree_builtin_log, 0 },
{ "ls", ostree_builtin_ls, 0 },
- { "pack", ostree_builtin_pack, 0 },
{ "prune", ostree_builtin_prune, 0 },
{ "pull", NULL, 0 },
{ "pull-local", ostree_builtin_pull_local, 0 },
{ "remote", ostree_builtin_remote, 0 },
{ "rev-parse", ostree_builtin_rev_parse, 0 },
{ "show", ostree_builtin_show, 0 },
- { "unpack", ostree_builtin_unpack, 0 },
{ "write-refs", ostree_builtin_write_refs, 0 },
{ NULL }
};
#include "ostree-fetcher.h"
gboolean verbose;
-gint opt_packfile_threshold = 66;
gboolean opt_related;
gint opt_depth;
static GOptionEntry options[] = {
{ "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose, "Show more information", NULL },
- { "packfile-threshold", 't', 0, G_OPTION_ARG_INT, &opt_packfile_threshold, "Only download packfiles if more than PERCENT objects are needed (default: 66)", "PERCENT" },
{ "related", 0, 0, G_OPTION_ARG_NONE, &opt_related, "Download related commits", NULL },
{ "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Download parent commits up to this depth (default: 0)", NULL },
{ NULL },
OstreeFetcher *fetcher;
SoupURI *base_uri;
- gboolean fetched_packs;
- GPtrArray *cached_meta_pack_indexes;
- GPtrArray *cached_data_pack_indexes;
-
GHashTable *file_checksums_to_fetch;
GMainLoop *loop;
return ret;
}
-static gboolean
-fetch_one_pack_file (OtPullData *pull_data,
- const char *pack_checksum,
- gboolean is_meta,
- GFile **out_cached_path,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *ret_cached_path = NULL;
- ot_lobj GFile *tmp_path = NULL;
- ot_lfree char *pack_name = NULL;
- SoupURI *pack_uri = NULL;
-
- if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
- pack_checksum, is_meta, &ret_cached_path,
- cancellable, error))
- goto out;
-
- if (ret_cached_path == NULL)
- {
- pack_name = ostree_get_pack_data_name (is_meta, pack_checksum);
- pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL);
-
- if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error))
- goto out;
-
- if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
- pack_checksum, is_meta, tmp_path,
- cancellable, error))
- goto out;
- }
-
- g_clear_object (&ret_cached_path);
- if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
- pack_checksum, is_meta, &ret_cached_path,
- cancellable, error))
- goto out;
-
- g_assert (ret_cached_path != NULL);
-
- ret = TRUE;
- ot_transfer_out_value (out_cached_path, &ret_cached_path);
- out:
- if (pack_uri)
- soup_uri_free (pack_uri);
- return ret;
-}
-
-static gboolean
-find_object_in_one_remote_pack (OtPullData *pull_data,
- GVariant *csum_bytes_v,
- OstreeObjectType objtype,
- const char *pack_checksum,
- gboolean *out_exists,
- guint64 *out_offset,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint64 ret_offset;
- gboolean ret_exists;
- ot_lvariant GVariant *mapped_pack = NULL;
-
- if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
- pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype),
- &mapped_pack,
- cancellable, error))
- goto out;
-
- ret_exists = ostree_pack_index_search (mapped_pack, csum_bytes_v, objtype, &ret_offset);
-
- ret = TRUE;
- if (out_exists)
- *out_exists = ret_exists;
- if (out_offset)
- *out_offset = ret_offset;
- out:
- return ret;
-}
-
-static gboolean
-find_object_in_remote_packs (OtPullData *pull_data,
- const char *checksum,
- OstreeObjectType objtype,
- char **out_pack_checksum,
- guint64 *out_offset,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint64 ret_offset = 0;
- guint i;
- GPtrArray *iter;
- ot_lvariant GVariant *csum_bytes_v = NULL;
- ot_lfree char *ret_pack_checksum = NULL;
-
- csum_bytes_v = ostree_checksum_to_bytes_v (checksum);
-
- if (OSTREE_OBJECT_TYPE_IS_META (objtype))
- iter = pull_data->cached_meta_pack_indexes;
- else
- iter = pull_data->cached_data_pack_indexes;
- for (i = 0; i < iter->len; i++)
- {
- const char *pack_checksum = iter->pdata[i];
- gboolean exists;
-
- if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, objtype,
- pack_checksum, &exists, &ret_offset,
- cancellable, error))
- goto out;
-
- if (exists)
- {
- ret_pack_checksum = g_strdup (pack_checksum);
- break;
- }
- }
-
- ret = TRUE;
- ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
- if (out_offset)
- *out_offset = ret_offset;
- out:
- return ret;
-}
-
-static gboolean
-fetch_one_cache_index (OtPullData *pull_data,
- const char *pack_checksum,
- gboolean is_meta,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *tmp_path = NULL;
- ot_lfree char *pack_index_name = NULL;
- SoupURI *index_uri = NULL;
-
- pack_index_name = ostree_get_pack_index_name (is_meta, pack_checksum);
- index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL);
-
- if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path,
- cancellable, error))
- goto out;
-
- if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
- pack_checksum, is_meta, tmp_path,
- cancellable, error))
- goto out;
-
- if (!ot_gfile_unlink (tmp_path, cancellable, error))
- goto out;
-
- g_clear_object (&tmp_path);
-
- ret = TRUE;
- out:
- if (tmp_path != NULL)
- (void) ot_gfile_unlink (tmp_path, NULL, NULL);
- if (index_uri)
- soup_uri_free (index_uri);
- return ret;
-}
-
-static gboolean
-fetch_and_cache_pack_indexes (OtPullData *pull_data,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- ot_lobj GFile *superindex_tmppath = NULL;
- ot_lptrarray GPtrArray *cached_meta_indexes = NULL;
- ot_lptrarray GPtrArray *cached_data_indexes = NULL;
- ot_lptrarray GPtrArray *uncached_meta_indexes = NULL;
- ot_lptrarray GPtrArray *uncached_data_indexes = NULL;
- ot_lvariant GVariant *superindex_variant = NULL;
- GVariantIter *contents_iter = NULL;
- SoupURI *superindex_uri = NULL;
-
- superindex_uri = suburi_new (pull_data->base_uri, "objects", "pack", "index", NULL);
-
- if (!fetch_uri (pull_data, superindex_uri, "index-",
- &superindex_tmppath, cancellable, error))
- goto out;
-
- if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name,
- superindex_tmppath,
- &cached_meta_indexes,
- &cached_data_indexes,
- &uncached_meta_indexes,
- &uncached_data_indexes,
- cancellable, error))
- goto out;
-
- for (i = 0; i < cached_meta_indexes->len; i++)
- g_ptr_array_add (pull_data->cached_meta_pack_indexes,
- g_strdup (cached_meta_indexes->pdata[i]));
- for (i = 0; i < cached_data_indexes->len; i++)
- g_ptr_array_add (pull_data->cached_data_pack_indexes,
- g_strdup (cached_data_indexes->pdata[i]));
-
- for (i = 0; i < uncached_meta_indexes->len; i++)
- {
- const char *pack_checksum = uncached_meta_indexes->pdata[i];
- if (!fetch_one_cache_index (pull_data, pack_checksum, TRUE, cancellable, error))
- goto out;
- g_ptr_array_add (pull_data->cached_meta_pack_indexes, g_strdup (pack_checksum));
- }
- for (i = 0; i < uncached_data_indexes->len; i++)
- {
- const char *pack_checksum = uncached_data_indexes->pdata[i];
- if (!fetch_one_cache_index (pull_data, pack_checksum, FALSE, cancellable, error))
- goto out;
- g_ptr_array_add (pull_data->cached_data_pack_indexes, g_strdup (pack_checksum));
- }
-
- ret = TRUE;
- out:
- if (superindex_uri)
- soup_uri_free (superindex_uri);
- if (contents_iter)
- g_variant_iter_free (contents_iter);
- return ret;
-}
-
static gboolean
fetch_loose_object (OtPullData *pull_data,
const char *checksum,
return ret;
}
-static gboolean
-find_object_ensure_indexes (OtPullData *pull_data,
- const char *checksum,
- OstreeObjectType objtype,
- gboolean *out_is_stored,
- char **out_remote_pack_checksum,
- guint64 *out_remote_pack_offset,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- gboolean ret_is_stored;
- ot_lfree char *ret_remote_pack_checksum = NULL;
-
- if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &ret_is_stored,
- cancellable, error))
- goto out;
-
- if (!ret_is_stored)
- {
- if (!pull_data->fetched_packs)
- {
- pull_data->fetched_packs = TRUE;
- pull_data->cached_meta_pack_indexes = g_ptr_array_new_with_free_func (g_free);
- pull_data->cached_data_pack_indexes = g_ptr_array_new_with_free_func (g_free);
-
- if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error))
- goto out;
- }
-
- if (!find_object_in_remote_packs (pull_data, checksum, objtype,
- &ret_remote_pack_checksum, out_remote_pack_offset,
- cancellable, error))
- goto out;
- }
-
- ret = TRUE;
- if (out_is_stored)
- *out_is_stored = ret_is_stored;
- ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum);
- out:
- return ret;
-}
-
-static gboolean
-find_object_ensure_pack_data (OtPullData *pull_data,
- const char *checksum,
- OstreeObjectType objtype,
- gboolean *out_is_stored,
- GFile **out_remote_pack_path,
- guint64 *out_remote_pack_offset,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- gboolean ret_is_stored;
- ot_lfree char *remote_pack_checksum = NULL;
- ot_lobj GFile *ret_remote_pack_path = NULL;
-
- if (!find_object_ensure_indexes (pull_data, checksum, objtype, &ret_is_stored,
- &remote_pack_checksum, out_remote_pack_offset,
- cancellable, error))
- goto out;
-
- if (!ret_is_stored)
- {
- if (remote_pack_checksum)
- {
- if (!fetch_one_pack_file (pull_data, remote_pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype),
- &ret_remote_pack_path, cancellable, error))
- goto out;
- }
- }
-
- ret = TRUE;
- if (out_is_stored)
- *out_is_stored = ret_is_stored;
- ot_transfer_out_value (out_remote_pack_path, &ret_remote_pack_path);
- /* offset set above */
- out:
- return ret;
-}
-
static gboolean
fetch_and_store_metadata (OtPullData *pull_data,
const char *checksum,
GError **error)
{
gboolean ret = FALSE;
- guint64 pack_offset = 0;
gboolean is_stored;
ot_lvariant GVariant *ret_variant = NULL;
- ot_lobj GFile *remote_pack_path = NULL;
ot_lobj GFile *temp_path = NULL;
ot_lobj GInputStream *input = NULL;
- ot_lvariant GVariant *pack_entry = NULL;
ot_lvariant GVariant *metadata = NULL;
- GMappedFile *pack_map = NULL;
g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype));
- if (!find_object_ensure_pack_data (pull_data, checksum, objtype,
- &is_stored, &remote_pack_path, &pack_offset,
- cancellable, error))
+ if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored,
+ cancellable, error))
goto out;
- if (remote_pack_path != NULL)
- {
- g_assert (!is_stored);
-
- pack_map = g_mapped_file_new (ot_gfile_get_path_cached (remote_pack_path), FALSE, error);
- if (!pack_map)
- goto out;
-
- if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
- g_mapped_file_get_length (pack_map),
- pack_offset, FALSE, TRUE, &pack_entry,
- cancellable, error))
- goto out;
-
- g_variant_get_child (pack_entry, 2, "v", &metadata);
-
- input = ot_variant_read (metadata);
- }
- else if (!is_stored)
+ if (!is_stored)
{
if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error))
goto out;
input = (GInputStream*)g_file_read (temp_path, cancellable, error);
if (!input)
goto out;
- }
- if (input)
- {
- g_assert (remote_pack_path != NULL || !is_stored);
if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, input,
cancellable, error))
goto out;
out:
if (temp_path)
(void) ot_gfile_unlink (temp_path, NULL, NULL);
- if (pack_map)
- g_mapped_file_unref (pack_map);
return ret;
}
ot_lvariant GVariant *files_variant = NULL;
ot_lvariant GVariant *dirs_variant = NULL;
ot_lobj GFile *stored_path = NULL;
- ot_lfree char *pack_checksum = NULL;
if (depth > OSTREE_MAX_RECURSION)
{
return ret;
}
-static gboolean
-store_file_from_pack (OtPullData *pull_data,
- const char *checksum,
- const char *pack_checksum,
- GFile *pack_file,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- gboolean exists;
- guint64 pack_offset;
- ot_lobj GFile *remote_pack_path = NULL;
- ot_lobj GFile *temp_path = NULL;
- ot_lvariant GVariant *pack_entry = NULL;
- ot_lobj GInputStream *input = NULL;
- ot_lobj GInputStream *file_object_input = NULL;
- ot_lobj GFileInfo *file_info = NULL;
- ot_lvariant GVariant *xattrs = NULL;
- ot_lvariant GVariant *csum_bytes_v = NULL;
- GMappedFile *pack_map = NULL;
-
- csum_bytes_v = ostree_checksum_to_bytes_v (checksum);
-
- if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, OSTREE_OBJECT_TYPE_FILE,
- pack_checksum, &exists, &pack_offset,
- cancellable, error))
- goto out;
-
- g_assert (exists);
-
- pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_file), FALSE, error);
- if (!pack_map)
- goto out;
-
- if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
- g_mapped_file_get_length (pack_map),
- pack_offset, FALSE, FALSE, &pack_entry,
- cancellable, error))
- goto out;
-
- if (!ostree_parse_file_pack_entry (pack_entry, &input, &file_info, &xattrs,
- cancellable, error))
- goto out;
-
- if (!ostree_raw_file_to_content_stream (input, file_info, xattrs,
- &file_object_input, NULL, cancellable, error))
- goto out;
-
- if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum,
- file_object_input,
- cancellable, error))
- goto out;
-
- ret = TRUE;
- out:
- if (pack_map)
- g_mapped_file_unref (pack_map);
- return ret;
-}
-
typedef struct {
OtPullData *pull_data;
}
if (!ostree_repo_stage_file_object_trusted (data->pull_data->repo, checksum,
- FALSE, file_object_input, length,
+ file_object_input, length,
cancellable, error))
goto out;
gpointer key, value;
ot_lobj GFile *temp_path = NULL;
ot_lobj GFile *content_temp_path = NULL;
- ot_lhash GHashTable *data_packs_to_fetch = NULL;
ot_lhash GHashTable *loose_files = NULL;
SoupURI *content_uri = NULL;
guint n_objects_to_fetch = 0;
- data_packs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free,
- (GDestroyNotify) g_ptr_array_unref);
loose_files = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_hash_table_iter_init (&hash_iter, pull_data->file_checksums_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *checksum = key;
- GPtrArray *files_to_fetch;
gboolean is_stored;
- ot_lfree char *remote_pack_checksum = NULL;
- if (!find_object_ensure_indexes (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE,
- &is_stored, &remote_pack_checksum, NULL,
- cancellable, error))
+ if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, &is_stored,
+ cancellable, error))
goto out;
- if (remote_pack_checksum)
- {
- files_to_fetch = g_hash_table_lookup (data_packs_to_fetch, remote_pack_checksum);
- if (files_to_fetch == NULL)
- {
- files_to_fetch = g_ptr_array_new_with_free_func (g_free);
- g_hash_table_insert (data_packs_to_fetch, remote_pack_checksum, files_to_fetch);
- /* transfer ownership */
- remote_pack_checksum = NULL;
- }
- g_ptr_array_add (files_to_fetch, g_strdup (checksum));
- n_objects_to_fetch++;
- }
- else if (!is_stored)
+ if (!is_stored)
{
char *key = g_strdup (checksum);
g_hash_table_insert (loose_files, key, key);
}
}
- g_hash_table_iter_init (&hash_iter, data_packs_to_fetch);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- const char *pack_checksum = key;
- GPtrArray *files = value;
- ot_lvariant GVariant *mapped_pack = NULL;
- ot_lvariant GVariant *content_list = NULL;
- gboolean fetch;
-
- if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
- pack_checksum, FALSE,
- &mapped_pack,
- cancellable, error))
- goto out;
-
- content_list = g_variant_get_child_value (mapped_pack, 2);
-
- fetch = (((double)files->len) / g_variant_n_children (content_list)) > ((double)opt_packfile_threshold / 100);
- if (!fetch)
- {
- guint i;
- for (i = 0; i < files->len; i++)
- {
- g_hash_table_insert (loose_files, files->pdata[i], files->pdata[i]);
- files->pdata[i] = NULL; /* steal data */
- }
- g_hash_table_iter_remove (&hash_iter);
- }
- }
-
if (n_objects_to_fetch > 0)
g_print ("%u content objects to fetch\n", n_objects_to_fetch);
- if (g_hash_table_size (data_packs_to_fetch) > 0)
- g_print ("Fetching %u content packs\n",
- g_hash_table_size (data_packs_to_fetch));
-
- g_hash_table_iter_init (&hash_iter, data_packs_to_fetch);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- const char *pack_checksum = key;
- GPtrArray *file_checksums = value;
- guint i;
- ot_lobj GFile *pack_path = NULL;
-
- if (!fetch_one_pack_file (pull_data, pack_checksum, FALSE,
- &pack_path, cancellable, error))
- goto out;
-
- g_print ("Storing %u objects from content pack %s\n", file_checksums->len,
- pack_checksum);
- for (i = 0; i < file_checksums->len; i++)
- {
- const char *checksum = file_checksums->pdata[i];
- if (!store_file_from_pack (pull_data, checksum, pack_checksum, pack_path,
- cancellable, error))
- goto out;
- }
-
- if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
- pack_checksum, FALSE, NULL,
- cancellable, error))
- goto out;
- }
-
if (g_hash_table_size (loose_files) > 0)
g_print ("Fetching %u loose objects\n",
g_hash_table_size (loose_files));
g_print ("remote %s is now %s\n", remote_ref, checksum);
}
- if (!ostree_repo_clean_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
- cancellable, error))
- goto out;
-
bytes_transferred = ostree_fetcher_bytes_transferred (pull_data->fetcher);
if (bytes_transferred > 0)
{
if (pull_data->base_uri)
soup_uri_free (pull_data->base_uri);
g_clear_pointer (&pull_data->file_checksums_to_fetch, (GDestroyNotify) g_hash_table_unref);
- g_clear_pointer (&pull_data->cached_meta_pack_indexes, (GDestroyNotify) g_ptr_array_unref);
- g_clear_pointer (&pull_data->cached_data_pack_indexes, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref);
if (summary_uri)
soup_uri_free (summary_uri);
typedef struct {
OstreeRepo *repo;
- guint n_pack_files;
} OtFsckData;
-static gboolean
-fsck_one_pack_file (OtFsckData *data,
- const char *pack_checksum,
- gboolean is_meta,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guchar objtype_u8;
- guint64 offset;
- guint64 pack_size;
- ot_lfree char *path = NULL;
- ot_lobj GFileInfo *pack_info = NULL;
- ot_lobj GInputStream *input = NULL;
- ot_lvariant GVariant *index_variant = NULL;
- ot_lobj GFile *pack_index_path = NULL;
- ot_lobj GFile *pack_data_path = NULL;
- ot_lfree guchar *pack_content_csum = NULL;
- ot_lfree char *tmp_checksum = NULL;
- GVariantIter *index_content_iter = NULL;
-
- g_free (path);
- path = ostree_get_relative_pack_index_path (is_meta, pack_checksum);
- pack_index_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path);
-
- if (!ot_util_variant_map (pack_index_path,
- OSTREE_PACK_INDEX_VARIANT_FORMAT, FALSE,
- &index_variant, error))
- goto out;
-
- if (!ostree_validate_structureof_pack_index (index_variant, error))
- goto out;
-
- g_free (path);
- path = ostree_get_relative_pack_data_path (is_meta, pack_checksum);
- pack_data_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path);
-
- input = (GInputStream*)g_file_read (pack_data_path, cancellable, error);
- if (!input)
- goto out;
-
- pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO,
- cancellable, error);
- if (!pack_info)
- goto out;
- pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size");
-
- if (!ot_gio_checksum_stream (input, &pack_content_csum, cancellable, error))
- goto out;
-
- tmp_checksum = ostree_checksum_from_bytes (pack_content_csum);
- if (strcmp (tmp_checksum, pack_checksum) != 0)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "corrupted pack '%s', actual checksum is %s",
- pack_checksum, tmp_checksum);
- goto out;
- }
-
- g_variant_get_child (index_variant, 2, "a(yayt)", &index_content_iter);
-
- while (g_variant_iter_loop (index_content_iter, "(y@ayt)",
- &objtype_u8, NULL, &offset))
- {
- offset = GUINT64_FROM_BE (offset);
- if (offset > pack_size)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT,
- pack_checksum,
- offset, pack_size);
- goto out;
- }
- }
-
- ret = TRUE;
- out:
- if (index_content_iter)
- g_variant_iter_free (index_content_iter);
- return ret;
-}
-
-static gboolean
-fsck_pack_files (OtFsckData *data,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- ot_lptrarray GPtrArray *meta_pack_indexes = NULL;
- ot_lptrarray GPtrArray *data_pack_indexes = NULL;
-
- if (!ostree_repo_list_pack_indexes (data->repo, &meta_pack_indexes, &data_pack_indexes,
- cancellable, error))
- goto out;
-
- for (i = 0; i < meta_pack_indexes->len; i++)
- {
- const char *pack_checksum = meta_pack_indexes->pdata[i];
-
- if (!fsck_one_pack_file (data, pack_checksum, TRUE, cancellable, error))
- goto out;
-
- data->n_pack_files++;
- }
- for (i = 0; i < data_pack_indexes->len; i++)
- {
- const char *pack_checksum = data_pack_indexes->pdata[i];
-
- if (!fsck_one_pack_file (data, pack_checksum, FALSE, cancellable, error))
- goto out;
-
- data->n_pack_files++;
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
static gboolean
fsck_reachable_objects_from_commits (OtFsckData *data,
GHashTable *commits,
if (!fsck_reachable_objects_from_commits (&data, commits, cancellable, error))
goto out;
- g_print ("Verifying structure of pack files...\n");
-
- if (!fsck_pack_files (&data, cancellable, error))
- goto out;
-
ret = TRUE;
out:
if (context)
{
GOptionContext *context = NULL;
gboolean ret = FALSE;
- GCancellable *cancellable = NULL;
+ __attribute__ ((unused)) GCancellable *cancellable = NULL;
ot_lobj GFile *child = NULL;
ot_lobj GFile *grandchild = NULL;
ot_lobj OstreeRepo *repo = NULL;
if (!ostree_repo_check (repo, error))
goto out;
- if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
- goto out;
-
ret = TRUE;
out:
if (context)
+++ /dev/null
-/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
- *
- * Copyright (C) 2012 Colin Walters <walters@verbum.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
- * Boston, MA 02111-1307, USA.
- *
- * Author: Colin Walters <walters@verbum.org>
- */
-
-#include "config.h"
-
-#include "ot-builtins.h"
-#include "ostree.h"
-
-#include <glib/gi18n.h>
-#include <glib/gprintf.h>
-
-#include <gio/gunixinputstream.h>
-#include <gio/gunixoutputstream.h>
-
-#define OT_GZIP_COMPRESSION_LEVEL (8)
-
-static gboolean opt_analyze_only;
-static gboolean opt_metadata_only;
-static gboolean opt_content_only;
-static gboolean opt_reindex_only;
-static gboolean opt_delete_all_loose;
-static gboolean opt_keep_all_loose;
-static char* opt_pack_size = "50m";
-static char* opt_int_compression;
-static char* opt_ext_compression;
-
-typedef enum {
- OT_COMPRESSION_NONE,
- OT_COMPRESSION_GZIP,
- OT_COMPRESSION_XZ
-} OtCompressionType;
-
-static GOptionEntry options[] = {
- { "pack-size", 0, 0, G_OPTION_ARG_STRING, &opt_pack_size, "Maximum uncompressed size of packfiles in bytes; may be suffixed with k, m, or g (default: 50m)", "BYTES" },
- { "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION", "COMPRESSION" },
- { "external-compression", 0, 0, G_OPTION_ARG_STRING, &opt_ext_compression, "Compress entire packfiles using COMPRESSION", "COMPRESSION" },
- { "metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_metadata_only, "Only pack metadata objects", NULL },
- { "content-only", 0, 0, G_OPTION_ARG_NONE, &opt_content_only, "Only pack content objects", NULL },
- { "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL },
- { "reindex-only", 0, 0, G_OPTION_ARG_NONE, &opt_reindex_only, "Regenerate pack index", NULL },
- { "delete-all-loose", 0, 0, G_OPTION_ARG_NONE, &opt_delete_all_loose, "Delete all loose objects (default: delete unreferenced loose)", NULL },
- { "keep-all-loose", 0, 0, G_OPTION_ARG_NONE, &opt_keep_all_loose, "Don't delete any loose objects (default: delete unreferenced loose)", NULL },
- { NULL }
-};
-
-typedef struct {
- OstreeRepo *repo;
-
- guint64 pack_size;
- OtCompressionType int_compression;
- OtCompressionType ext_compression;
-
- gboolean had_error;
- GError **error;
-} OtRepackData;
-
-typedef struct {
- GOutputStream *out;
- GPtrArray *compressor_argv;
- GPid compress_child_pid;
-} OtBuildRepackFile;
-
-static gint
-compare_object_data_by_size (gconstpointer ap,
- gconstpointer bp)
-{
- GVariant *a = *(void **)ap;
- GVariant *b = *(void **)bp;
- guint64 a_size;
- guint64 b_size;
-
- g_variant_get_child (a, 2, "t", &a_size);
- g_variant_get_child (b, 2, "t", &b_size);
- if (a == b)
- return 0;
- else if (a > b)
- return 1;
- else
- return -1;
-}
-
-static gboolean
-write_bytes_update_checksum (GOutputStream *output,
- gconstpointer bytes,
- gsize len,
- GChecksum *checksum,
- guint64 *inout_offset,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- gsize bytes_written;
-
- if (len > 0)
- {
- g_checksum_update (checksum, (guchar*) bytes, len);
- if (!g_output_stream_write_all (output, bytes, len, &bytes_written,
- cancellable, error))
- goto out;
- g_assert_cmpint (bytes_written, ==, len);
- *inout_offset += bytes_written;
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-static gboolean
-write_padding (GOutputStream *output,
- guint alignment,
- GChecksum *checksum,
- guint64 *inout_offset,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint bits;
- guint padding_len;
- guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0};
-
- if (alignment == 8)
- bits = ((*inout_offset) & 7);
- else
- bits = ((*inout_offset) & 3);
-
- if (bits > 0)
- {
- padding_len = alignment - bits;
- if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding_len,
- checksum, inout_offset, cancellable, error))
- goto out;
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-static gint
-compare_index_content (gconstpointer ap,
- gconstpointer bp)
-{
- gpointer a = *((gpointer*)ap);
- gpointer b = *((gpointer*)bp);
- GVariant *a_v = a;
- GVariant *b_v = b;
- guchar a_objtype;
- guchar b_objtype;
- guint64 a_offset;
- guint64 b_offset;
- int c;
- ot_lvariant GVariant *a_csum_bytes = NULL;
- ot_lvariant GVariant *b_csum_bytes = NULL;
-
- g_variant_get (a_v, "(y@ayt)", &a_objtype, &a_csum_bytes, &a_offset);
- g_variant_get (b_v, "(y@ayt)", &b_objtype, &b_csum_bytes, &b_offset);
- c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (a_csum_bytes),
- ostree_checksum_bytes_peek (b_csum_bytes));
- if (c == 0)
- {
- if (a_objtype < b_objtype)
- c = -1;
- else if (a_objtype > b_objtype)
- c = 1;
- }
- return c;
-}
-
-static gboolean
-delete_loose_object (OtRepackData *data,
- const char *checksum,
- OstreeObjectType objtype,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- gboolean do_delete = FALSE;
- GError *temp_error = NULL;
- ot_lobj GFile *object_path = NULL;
- ot_lobj GFile *file_content_object_path = NULL;
-
- object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
-
- if (objtype == OSTREE_OBJECT_TYPE_FILE)
- {
- ot_lobj GFileInfo *file_info = NULL;
-
- if (ostree_repo_get_mode (data->repo) == OSTREE_REPO_MODE_BARE)
- {
- file_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- cancellable, &temp_error);
- }
- else
- {
- ot_lobj GFile *content_object_path = NULL;
-
- content_object_path = ostree_repo_get_archive_content_path (data->repo, checksum);
-
- file_info = g_file_query_info (content_object_path, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- cancellable, &temp_error);
- }
-
- if (!file_info)
- {
- if (!g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
- {
- g_propagate_error (error, temp_error);
- goto out;
- }
- else
- {
- g_clear_error (&temp_error);
- }
- }
-
- do_delete = opt_delete_all_loose
- || (file_info && g_file_info_get_attribute_uint32 (file_info, "unix::nlink") <= 1);
- }
- else
- do_delete = TRUE;
-
- if (do_delete)
- {
- if (!ot_gfile_unlink (object_path, cancellable, error))
- {
- g_prefix_error (error, "Failed to delete loose object '%s'",
- ot_gfile_get_path_cached (object_path));
- goto out;
- }
-
- if (objtype == OSTREE_OBJECT_TYPE_FILE
- && ostree_repo_get_mode (data->repo) == OSTREE_REPO_MODE_ARCHIVE)
- {
- ot_lobj GFile *content_object_path = NULL;
-
- content_object_path = ostree_repo_get_archive_content_path (data->repo, checksum);
-
- /* Ignoring errors for now; later should only be trying to
- * delete files with content.
- */
- (void) ot_gfile_unlink (content_object_path, NULL, NULL);
- }
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-static gboolean
-pack_one_meta_object (OtRepackData *data,
- const char *checksum,
- OstreeObjectType objtype,
- GVariant **out_packed_object,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GFile *object_path = NULL;
- ot_lvariant GVariant *metadata_v = NULL;
- ot_lvariant GVariant *ret_packed_object = NULL;
-
- object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
-
- if (!ot_util_variant_map (object_path, ostree_metadata_variant_type (objtype),
- TRUE, &metadata_v, error))
- goto out;
-
- ret_packed_object = g_variant_new ("(y@ayv)", (guchar) objtype,
- ostree_checksum_to_bytes_v (checksum),
- metadata_v);
-
- ret = TRUE;
- ot_transfer_out_value (out_packed_object, &ret_packed_object);
- out:
- return ret;
-}
-
-static gboolean
-pack_one_data_object (OtRepackData *data,
- const char *checksum,
- OstreeObjectType objtype,
- guint64 expected_objsize,
- GVariant **out_packed_object,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guchar entry_flags = 0;
- GInputStream *read_object_in; /* nofree */
- ot_lobj GInputStream *input = NULL;
- ot_lobj GFileInfo *file_info = NULL;
- ot_lvariant GVariant *xattrs = NULL;
- ot_lobj GMemoryOutputStream *object_data_stream = NULL;
- ot_lobj GConverter *compressor = NULL;
- ot_lobj GConverterInputStream *compressed_object_input = NULL;
- ot_lvariant GVariant *file_header = NULL;
- ot_lvariant GVariant *ret_packed_object = NULL;
-
- switch (data->int_compression)
- {
- case OT_COMPRESSION_GZIP:
- {
- entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
- break;
- }
- default:
- {
- g_assert_not_reached ();
- }
- }
-
- if (!ostree_repo_load_file (data->repo, checksum, &input, &file_info, &xattrs,
- cancellable, error))
- goto out;
-
- file_header = ostree_file_header_new (file_info, xattrs);
-
- object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
- if (input != NULL)
- {
- if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
- {
- compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL);
- compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
- "converter", compressor,
- "base-stream", input,
- "close-base-stream", TRUE,
- NULL);
- read_object_in = (GInputStream*)compressed_object_input;
- }
- else
- {
- read_object_in = (GInputStream*)input;
- }
-
- if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in,
- G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
- cancellable, error))
- goto out;
- }
-
- {
- guchar *data = g_memory_output_stream_get_data (object_data_stream);
- gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
- ret_packed_object = g_variant_new ("(@ayy@(uuuusa(ayay))@ay)",
- ostree_checksum_to_bytes_v (checksum),
- entry_flags,
- file_header,
- ot_gvariant_new_bytearray (data, data_len));
- }
-
- ret = TRUE;
- ot_transfer_out_value (out_packed_object, &ret_packed_object);
- out:
- return ret;
-}
-
-static gboolean
-create_pack_file (OtRepackData *data,
- gboolean is_meta,
- GPtrArray *objects,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- guint64 offset;
- gsize bytes_written;
- ot_lobj GFile *pack_dir = NULL;
- ot_lobj GFile *index_temppath = NULL;
- ot_lobj GOutputStream *index_out = NULL;
- ot_lobj GFile *pack_temppath = NULL;
- ot_lobj GOutputStream *pack_out = NULL;
- ot_lptrarray GPtrArray *index_content_list = NULL;
- ot_lvariant GVariant *pack_header = NULL;
- ot_lvariant GVariant *index_content = NULL;
- ot_lfree char *pack_name = NULL;
- ot_lobj GFile *pack_file_path = NULL;
- ot_lobj GFile *pack_index_path = NULL;
- GVariantBuilder index_content_builder;
- GChecksum *pack_checksum = NULL;
-
- if (g_cancellable_set_error_if_cancelled (cancellable, error))
- return FALSE;
-
- if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
- "pack-index", NULL,
- &index_temppath,
- &index_out,
- cancellable, error))
- goto out;
-
- if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
- "pack-content", NULL,
- &pack_temppath,
- &pack_out,
- cancellable, error))
- goto out;
-
- index_content_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
-
- offset = 0;
- pack_checksum = g_checksum_new (G_CHECKSUM_SHA256);
-
- pack_header = g_variant_new ("(s@a{sv}t)",
- is_meta ? "OSTv0PACKMETAFILE" : "OSTv0PACKDATAFILE",
- g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
- (guint64)objects->len);
-
- if (!ostree_write_variant_with_size (pack_out, pack_header, offset, &bytes_written, pack_checksum,
- cancellable, error))
- goto out;
- offset += bytes_written;
-
- for (i = 0; i < objects->len; i++)
- {
- GVariant *object_data = objects->pdata[i];
- const char *checksum;
- guint32 objtype_u32;
- OstreeObjectType objtype;
- guint64 expected_objsize;
- ot_lvariant GVariant *packed_object = NULL;
- ot_lvariant GVariant *index_entry = NULL;
-
- g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
-
- objtype = (OstreeObjectType) objtype_u32;
-
- if (is_meta)
- {
- if (!pack_one_meta_object (data, checksum, objtype, &packed_object,
- cancellable, error))
- goto out;
- }
- else
- {
- if (!pack_one_data_object (data, checksum, objtype, expected_objsize,
- &packed_object, cancellable, error))
- goto out;
- }
-
- if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
- goto out;
-
- /* offset points to aligned header size */
- index_entry = g_variant_new ("(y@ayt)",
- (guchar)objtype,
- ostree_checksum_to_bytes_v (checksum),
- GUINT64_TO_BE (offset));
- g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry));
- index_entry = NULL;
-
- bytes_written = 0;
- if (!ostree_write_variant_with_size (pack_out, packed_object, offset, &bytes_written,
- pack_checksum, cancellable, error))
- goto out;
- offset += bytes_written;
- }
-
- if (!g_output_stream_close (pack_out, cancellable, error))
- goto out;
-
- g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(yayt)"));
- g_ptr_array_sort (index_content_list, compare_index_content);
- for (i = 0; i < index_content_list->len; i++)
- {
- GVariant *index_item = index_content_list->pdata[i];
- g_variant_builder_add_value (&index_content_builder, index_item);
- }
- index_content = g_variant_new ("(s@a{sv}@a(yayt))",
- "OSTv0PACKINDEX",
- g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
- g_variant_builder_end (&index_content_builder));
-
- if (!g_output_stream_write_all (index_out,
- g_variant_get_data (index_content),
- g_variant_get_size (index_content),
- &bytes_written,
- cancellable,
- error))
- goto out;
-
- if (!g_output_stream_close (index_out, cancellable, error))
- goto out;
-
- if (!ostree_repo_add_pack_file (data->repo,
- g_checksum_get_string (pack_checksum),
- is_meta,
- index_temppath,
- pack_temppath,
- cancellable,
- error))
- goto out;
-
- if (!ostree_repo_regenerate_pack_index (data->repo, cancellable, error))
- goto out;
-
- g_print ("Created %s pack file '%s' with %u objects\n", is_meta ? "metadata" : "content",
- g_checksum_get_string (pack_checksum), objects->len);
-
- if (!opt_keep_all_loose)
- {
- for (i = 0; i < objects->len; i++)
- {
- GVariant *object_data = objects->pdata[i];
- const char *checksum;
- guint32 objtype_u32;
- OstreeObjectType objtype;
- guint64 expected_objsize;
-
- g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
-
- objtype = (OstreeObjectType) objtype_u32;
-
- if (!delete_loose_object (data, checksum, objtype, cancellable, error))
- goto out;
- }
- }
-
- ret = TRUE;
- out:
- if (index_temppath)
- (void) unlink (ot_gfile_get_path_cached (index_temppath));
- if (pack_temppath)
- (void) unlink (ot_gfile_get_path_cached (pack_temppath));
- if (pack_checksum)
- g_checksum_free (pack_checksum);
- return ret;
-}
-
-static void
-cluster_one_object_chain (OtRepackData *data,
- GPtrArray *object_list,
- GPtrArray *inout_clusters)
-{
- guint i;
- guint64 current_size;
- guint current_offset;
-
- current_size = 0;
- current_offset = 0;
- for (i = 0; i < object_list->len; i++)
- {
- GVariant *objdata = object_list->pdata[i];
- guint64 objsize;
-
- g_variant_get_child (objdata, 2, "t", &objsize);
-
- if (current_size + objsize > data->pack_size || i == (object_list->len - 1))
- {
- guint j;
- GPtrArray *current;
-
- if (current_offset < i)
- {
- current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
- for (j = current_offset; j <= i; j++)
- {
- g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
- }
- g_ptr_array_add (inout_clusters, current);
- current_size = objsize;
- current_offset = i+1;
- }
- }
- else if (objsize > data->pack_size)
- {
- break;
- }
- else
- {
- current_size += objsize;
- }
- }
-}
-
-/**
- * cluster_objects_stupidly:
- * @objects: Map from serialized object name to objdata
- * @out_meta_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
- * @out_data_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
- *
- * Just sorts by size currently. Also filters out non-regular object
- * content.
- */
-static gboolean
-cluster_objects_stupidly (OtRepackData *data,
- GHashTable *objects,
- GPtrArray **out_meta_clusters,
- GPtrArray **out_data_clusters,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- GHashTableIter hash_iter;
- gpointer key, value;
- ot_lptrarray GPtrArray *ret_meta_clusters = NULL;
- ot_lptrarray GPtrArray *ret_data_clusters = NULL;
- ot_lptrarray GPtrArray *meta_object_list = NULL;
- ot_lptrarray GPtrArray *data_object_list = NULL;
-
- meta_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
- data_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
-
- g_hash_table_iter_init (&hash_iter, objects);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- GVariant *serialized_key = key;
- const char *checksum;
- OstreeObjectType objtype;
- guint64 size;
- GVariant *v;
- ot_lobj GFile *object_path = NULL;
- ot_lobj GFileInfo *object_info = NULL;
-
- ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
-
- if (OSTREE_OBJECT_TYPE_IS_META (objtype))
- {
- object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
-
- object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- cancellable, error);
- if (!object_info)
- goto out;
- }
- else
- {
- if (!ostree_repo_load_file (data->repo, checksum, NULL, &object_info, NULL,
- cancellable, error))
- goto out;
- }
-
- size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
-
- v = g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size));
- if (OSTREE_OBJECT_TYPE_IS_META (objtype))
- g_ptr_array_add (meta_object_list, v);
- else
- g_ptr_array_add (data_object_list, v);
- }
-
- g_ptr_array_sort (meta_object_list, compare_object_data_by_size);
- g_ptr_array_sort (data_object_list, compare_object_data_by_size);
-
- ret_meta_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
- ret_data_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
-
- cluster_one_object_chain (data, meta_object_list, ret_meta_clusters);
- cluster_one_object_chain (data, data_object_list, ret_data_clusters);
-
- ret = TRUE;
- ot_transfer_out_value (out_meta_clusters, &ret_meta_clusters);
- ot_transfer_out_value (out_data_clusters, &ret_data_clusters);
- out:
- return ret;
-}
-
-static gboolean
-parse_size_spec_with_suffix (const char *spec,
- guint64 *out_size,
- GError **error)
-{
- gboolean ret = FALSE;
- char *endptr = NULL;
- guint64 ret_size;
-
- ret_size = g_ascii_strtoull (spec, &endptr, 10);
-
- if (endptr && *endptr)
- {
- char suffix = *endptr;
-
- switch (suffix)
- {
- case 'k':
- case 'K':
- {
- ret_size *= 1024;
- break;
- }
- case 'm':
- case 'M':
- {
- ret_size *= (1024 * 1024);
- break;
- }
- case 'g':
- case 'G':
- {
- ret_size *= (1024 * 1024 * 1024);
- break;
- }
- default:
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid size suffix '%c'", suffix);
- goto out;
- }
- }
-
- ret = TRUE;
- *out_size = ret_size;
- out:
- return ret;
-}
-
-static gboolean
-parse_compression_string (const char *compstr,
- OtCompressionType *out_comptype,
- GError **error)
-{
- gboolean ret = FALSE;
- OtCompressionType ret_comptype;
-
- if (compstr == NULL)
- ret_comptype = OT_COMPRESSION_NONE;
- else if (strcmp (compstr, "gzip") == 0)
- ret_comptype = OT_COMPRESSION_GZIP;
- else if (strcmp (compstr, "xz") == 0)
- ret_comptype = OT_COMPRESSION_XZ;
- else
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid compression '%s'", compstr);
- goto out;
- }
-
- ret = TRUE;
- *out_comptype = ret_comptype;
- out:
- return ret;
-}
-
-static gboolean
-do_stats_gather_loose (OtRepackData *data,
- GHashTable *objects,
- GHashTable **out_loose,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint n_loose = 0;
- guint n_loose_and_packed = 0;
- guint n_packed = 0;
- guint n_dup_packed = 0;
- guint n_commits = 0;
- guint n_dirmeta = 0;
- guint n_dirtree = 0;
- guint n_files = 0;
- GHashTableIter hash_iter;
- gpointer key, value;
- ot_lhash GHashTable *ret_loose = NULL;
-
- ret_loose = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
- (GDestroyNotify) g_variant_unref,
- NULL);
-
- g_hash_table_iter_init (&hash_iter, objects);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- GVariant *serialized_key = key;
- GVariant *objdata = value;
- const char *checksum;
- OstreeObjectType objtype;
- gboolean is_loose;
- gboolean is_packed;
- ot_lvariant GVariant *pack_array = NULL;
-
- ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
-
- g_variant_get (objdata, "(b@as)", &is_loose, &pack_array);
-
- is_packed = g_variant_n_children (pack_array) > 0;
-
- if (is_loose && is_packed)
- {
- n_loose_and_packed++;
- }
- else if (is_loose)
- {
- if (!(opt_metadata_only && !OSTREE_OBJECT_TYPE_IS_META(objtype))
- && !(opt_content_only && OSTREE_OBJECT_TYPE_IS_META(objtype)))
- {
- GVariant *copy = g_variant_ref (serialized_key);
- g_hash_table_replace (ret_loose, copy, copy);
- }
- n_loose++;
- }
- else if (g_variant_n_children (pack_array) > 1)
- {
- n_dup_packed++;
- }
- else
- {
- n_packed++;
- }
-
- switch (objtype)
- {
- case OSTREE_OBJECT_TYPE_COMMIT:
- n_commits++;
- break;
- case OSTREE_OBJECT_TYPE_DIR_TREE:
- n_dirtree++;
- break;
- case OSTREE_OBJECT_TYPE_DIR_META:
- n_dirmeta++;
- break;
- case OSTREE_OBJECT_TYPE_FILE:
- n_files++;
- break;
- default:
- g_assert_not_reached ();
- }
- }
-
- g_print ("Commits: %u\n", n_commits);
- g_print ("Tree contents: %u\n", n_dirtree);
- g_print ("Tree meta: %u\n", n_dirmeta);
- g_print ("Files: %u\n", n_files);
- g_print ("\n");
- g_print ("Loose+packed objects: %u\n", n_loose_and_packed);
- g_print ("Loose-only objects: %u\n", n_loose);
- g_print ("Duplicate packed objects: %u\n", n_dup_packed);
- g_print ("Packed-only objects: %u\n", n_packed);
-
- ret = TRUE;
- ot_transfer_out_value (out_loose, &ret_loose);
- /* out: */
- return ret;
-}
-
-static gboolean
-do_incremental_pack (OtRepackData *data,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint i;
- ot_lhash GHashTable *objects = NULL;
- ot_lptrarray GPtrArray *meta_clusters = NULL;
- ot_lptrarray GPtrArray *data_clusters = NULL;
- ot_lhash GHashTable *loose_objects = NULL;
-
- if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
- cancellable, error))
- goto out;
-
- if (!do_stats_gather_loose (data, objects, &loose_objects, cancellable, error))
- goto out;
-
- g_print ("\n");
- g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size);
-
- if (!cluster_objects_stupidly (data, loose_objects, &meta_clusters, &data_clusters,
- cancellable, error))
- goto out;
-
- if (meta_clusters->len > 0 || data_clusters->len > 0)
- g_print ("Going to create %u meta packfiles, %u data packfiles\n",
- meta_clusters->len, data_clusters->len);
- else
- g_print ("Nothing to do\n");
-
- if (!opt_analyze_only)
- {
- for (i = 0; i < meta_clusters->len; i++)
- {
- GPtrArray *cluster = meta_clusters->pdata[i];
-
- if (!create_pack_file (data, TRUE, cluster, cancellable, error))
- goto out;
- }
- for (i = 0; i < data_clusters->len; i++)
- {
- GPtrArray *cluster = data_clusters->pdata[i];
-
- if (!create_pack_file (data, FALSE, cluster, cancellable, error))
- goto out;
- }
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-gboolean
-ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error)
-{
- gboolean ret = FALSE;
- GOptionContext *context;
- GCancellable *cancellable = NULL;
- OtRepackData data;
- ot_lobj OstreeRepo *repo = NULL;
-
- memset (&data, 0, sizeof (data));
-
- context = g_option_context_new ("- Recompress objects");
- g_option_context_add_main_entries (context, options, NULL);
-
- if (!g_option_context_parse (context, &argc, &argv, error))
- goto out;
-
- if (opt_metadata_only && opt_content_only)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "--content-only cannot be specified with --metadata-only");
- goto out;
- }
-
- repo = ostree_repo_new (repo_path);
- if (!ostree_repo_check (repo, error))
- goto out;
-
- data.repo = repo;
- data.error = error;
-
- if (!parse_size_spec_with_suffix (opt_pack_size, &data.pack_size, error))
- goto out;
- /* Default internal compression to gzip */
- if (!parse_compression_string (opt_int_compression ? opt_int_compression : "gzip", &data.int_compression, error))
- goto out;
- if (!parse_compression_string (opt_ext_compression, &data.ext_compression, error))
- goto out;
-
- if (opt_reindex_only)
- {
- if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
- goto out;
- }
- else
- {
- if (!do_incremental_pack (&data, cancellable, error))
- goto out;
- }
-
- ret = TRUE;
- out:
- if (context)
- g_option_context_free (context);
- return ret;
-}
cancellable, error))
goto out;
- if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum, FALSE,
+ if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum,
file_object, length,
cancellable, error))
goto out;
input = ot_variant_read (metadata);
if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype,
- checksum, FALSE, input,
+ checksum, input,
cancellable, error))
goto out;
}
+++ /dev/null
-/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
- *
- * Copyright (C) 2012 Colin Walters <walters@verbum.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
- * Boston, MA 02111-1307, USA.
- *
- * Author: Colin Walters <walters@verbum.org>
- */
-
-#include "config.h"
-
-#include "ot-builtins.h"
-#include "ostree.h"
-
-#include <glib/gi18n.h>
-#include <glib/gprintf.h>
-
-#include <gio/gunixinputstream.h>
-#include <gio/gunixoutputstream.h>
-
-static gboolean opt_keep_packs;
-
-static GOptionEntry options[] = {
- { "keep-packs", 0, 0, G_OPTION_ARG_NONE, &opt_keep_packs, "Don't delete pack files", NULL },
- { NULL }
-};
-
-typedef struct {
- OstreeRepo *repo;
-} OtUnpackData;
-
-static gboolean
-unpack_one_object (OstreeRepo *repo,
- const char *checksum,
- OstreeObjectType objtype,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lobj GInputStream *input = NULL;
- ot_lobj GFileInfo *file_info = NULL;
- ot_lvariant GVariant *xattrs = NULL;
- ot_lvariant GVariant *meta = NULL;
-
- if (objtype == OSTREE_OBJECT_TYPE_FILE)
- {
- ot_lobj GInputStream *file_object = NULL;
- guint64 length;
-
- if (!ostree_repo_load_file (repo, checksum,
- &input, &file_info, &xattrs,
- cancellable, error))
- goto out;
-
- if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, &length,
- cancellable, error))
- goto out;
-
- if (!ostree_repo_stage_file_object_trusted (repo, checksum, TRUE, file_object, length,
- cancellable, error))
- goto out;
- }
- else
- {
- if (!ostree_repo_load_variant (repo, objtype, checksum, &meta, error))
- goto out;
-
- input = ot_variant_read (meta);
-
- if (!ostree_repo_stage_object_trusted (repo, objtype, checksum, TRUE,
- input, cancellable, error))
- goto out;
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-static gboolean
-delete_one_packfile (OstreeRepo *repo,
- const char *pack_checksum,
- gboolean is_meta,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lfree char *data_name = NULL;
- ot_lobj GFile *data_path = NULL;
- ot_lfree char *index_name = NULL;
- ot_lobj GFile *index_path = NULL;
-
- index_name = ostree_get_relative_pack_index_path (is_meta, pack_checksum);
- index_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), index_name);
- data_name = ostree_get_relative_pack_data_path (is_meta, pack_checksum);
- data_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), data_name);
-
- if (!ot_gfile_unlink (index_path, cancellable, error))
- {
- g_prefix_error (error, "Failed to delete pack index '%s': ", ot_gfile_get_path_cached (index_path));
- goto out;
- }
- if (!ot_gfile_unlink (data_path, cancellable, error))
- {
- g_prefix_error (error, "Failed to delete pack data '%s': ", ot_gfile_get_path_cached (data_path));
- goto out;
- }
-
- ret = TRUE;
- out:
- return ret;
-}
-
-gboolean
-ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
-{
- gboolean ret = FALSE;
- GOptionContext *context;
- GCancellable *cancellable = NULL;
- gboolean in_transaction = FALSE;
- OtUnpackData data;
- gpointer key, value;
- guint64 unpacked_object_count = 0;
- GHashTableIter hash_iter;
- ot_lobj OstreeRepo *repo = NULL;
- ot_lhash GHashTable *objects = NULL;
- ot_lptrarray GPtrArray *clusters = NULL;
- ot_lhash GHashTable *meta_packfiles_to_delete = NULL;
- ot_lhash GHashTable *data_packfiles_to_delete = NULL;
- ot_lobj GFile *objpath = NULL;
-
- memset (&data, 0, sizeof (data));
-
- context = g_option_context_new ("- Uncompress objects");
- g_option_context_add_main_entries (context, options, NULL);
-
- if (!g_option_context_parse (context, &argc, &argv, error))
- goto out;
-
- repo = ostree_repo_new (repo_path);
- if (!ostree_repo_check (repo, error))
- goto out;
-
- if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
- "Can't unpack bare repositories yet");
- goto out;
- }
-
- data.repo = repo;
-
- if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error))
- goto out;
-
- if (!ostree_repo_prepare_transaction (repo, cancellable, error))
- goto out;
-
- in_transaction = TRUE;
-
- meta_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
- data_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
-
- g_hash_table_iter_init (&hash_iter, objects);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- GVariant *objkey = key;
- GVariant *objdata;
- const char *checksum;
- const char *pack_checksum;
- OstreeObjectType objtype;
- gboolean is_loose;
- gboolean is_packed = FALSE;
- GVariantIter *pack_array_iter;
- GHashTable *target_hash;
-
- ostree_object_name_deserialize (objkey, &checksum, &objtype);
-
- objdata = g_hash_table_lookup (objects, objkey);
- g_assert (objdata);
-
- g_variant_get (objdata, "(bas)", &is_loose, &pack_array_iter);
-
- if (OSTREE_OBJECT_TYPE_IS_META (objtype))
- target_hash = meta_packfiles_to_delete;
- else
- target_hash = data_packfiles_to_delete;
-
- while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum))
- {
- is_packed = TRUE;
- if (!g_hash_table_lookup (target_hash, pack_checksum))
- {
- gchar *duped_checksum = g_strdup (pack_checksum);
- g_hash_table_replace (target_hash, duped_checksum, duped_checksum);
- }
- }
- g_variant_iter_free (pack_array_iter);
-
- if (is_packed)
- {
- if (!unpack_one_object (repo, checksum, objtype, cancellable, error))
- goto out;
-
- unpacked_object_count++;
- }
- }
-
- if (!ostree_repo_commit_transaction (repo, cancellable, error))
- goto out;
-
- if (!opt_keep_packs)
- {
- if (g_hash_table_size (meta_packfiles_to_delete) == 0
- && g_hash_table_size (data_packfiles_to_delete) == 0)
- g_print ("No pack files; nothing to do\n");
-
- g_hash_table_iter_init (&hash_iter, meta_packfiles_to_delete);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- const char *pack_checksum = key;
-
- if (!delete_one_packfile (repo, pack_checksum, TRUE, cancellable, error))
- goto out;
-
- g_print ("Deleted packfile '%s'\n", pack_checksum);
- }
-
- g_hash_table_iter_init (&hash_iter, data_packfiles_to_delete);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
- {
- const char *pack_checksum = key;
-
- if (!delete_one_packfile (repo, pack_checksum, FALSE, cancellable, error))
- goto out;
-
- g_print ("Deleted packfile '%s'\n", pack_checksum);
- }
-
- if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
- goto out;
- }
-
- g_print ("Unpacked %" G_GUINT64_FORMAT " objects\n", unpacked_object_count);
-
- ret = TRUE;
- out:
- if (in_transaction)
- (void) ostree_repo_abort_transaction (repo, cancellable, NULL);
- if (context)
- g_option_context_free (context);
- return ret;
-}
gboolean ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_show (int argc, char **argv, GFile *repo_path, GError **error);
-gboolean ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_rev_parse (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_remote (int argc, char **argv, GFile *repo_path, GError **error);
-gboolean ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_write_refs (int argc, char **argv, GFile *repo_path, GError **error);
G_END_DECLS
. libtest.sh
-echo '1..22'
+echo '1..11'
setup_test_repository "archive"
echo "ok setup"
assert_file_has_content cow-contents "moo"
echo "ok cat-file"
-cd ${test_tmpdir}
-$OSTREE pack
-echo "ok pack"
-
-cd ${test_tmpdir}
-$OSTREE fsck
-echo "ok fsck"
-
-$OSTREE checkout test2 checkout-test2-from-packed
-echo "ok checkout union 1"
-
cd ${test_tmpdir}
$OSTREE fsck
echo "ok fsck"
-
-$OSTREE pack --analyze-only
-echo "ok pack analyze"
-
-$OSTREE unpack
-echo "ok unpack"
-
-cd ${test_tmpdir}
-$OSTREE fsck
-echo "ok fsck"
-
-cd ${test_tmpdir}
-$OSTREE checkout test2 checkout-test2-from-unpacked
-echo "ok checkout union 2"
-
-$OSTREE pack --metadata-only
-echo "ok pack metadata"
-
-$OSTREE fsck
-echo "ok fsck"
-
-cd ${test_tmpdir}
-rm -rf checkout-test2
-$OSTREE checkout test2 checkout-test2
-echo "ok checkout metadata-packed"
-
-$OSTREE unpack
-echo "ok unpack"
. libtest.sh
-echo '1..4'
+echo '1..2'
setup_fake_remote_repo1
cd ${test_tmpdir}
assert_file_has_content firstfile '^first$'
assert_file_has_content baz/cow '^moo$'
echo "ok pull contents"
-
-cd ${test_tmpdir}
-ostree --repo=$(pwd)/ostree-srv/gnomerepo pack
-rm -rf repo
-mkdir repo
-${CMD_PREFIX} ostree --repo=repo init
-${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo
-${CMD_PREFIX} ostree-pull --repo=repo origin main
-${CMD_PREFIX} ostree --repo=repo fsck
-echo "ok pull packed"
-
-cd ${test_tmpdir}
-rm -rf checkout-origin-main
-$OSTREE checkout origin/main checkout-origin-main
-cd checkout-origin-main
-assert_file_has_content firstfile '^first$'
-assert_file_has_content baz/cow '^moo$'
-echo "ok pull contents packed"
projects / ostree.git / commitdiff